level=info ts=2026-05-13T20:47:32.856221755Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-13T20:47:32.856257956Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-13T20:47:32.856629069Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-13T20:47:32.857475513Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-13T20:47:32.8579232Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T20:47:32.858009605Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.75 inf=eth0
level=info ts=2026-05-13T20:47:32.858650927Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-76d5aec9
ts=2026-05-13T20:47:32.858759304Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-13T20:47:32.858775465Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.75 advertisePort=7946
level=debug ts=2026-05-13T20:47:32.858796022Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.75 advertisePort=7946
level=info ts=2026-05-13T20:47:32.858926728Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-13T20:47:32.863427222Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T20:47:32.863472264Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-13T20:47:32.864516972Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-13T20:47:32.864561833Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T20:47:32.865168533Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-13T20:47:32.865192207Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-13T20:47:32.865190524Z
level=debug ts=2026-05-13T20:47:32.865202381Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-13T20:47:32.865222163Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-13T20:47:32.865234289Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-13T20:47:32.865239078Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-13T20:47:32.865238114Z
level=debug ts=2026-05-13T20:47:32.865243585Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-05-13T20:47:32.865922913Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T20:47:32.865997669Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T20:47:32.866053419Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.127325ms
level=error ts=2026-05-13T20:47:32.866212985Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-13T20:47:32.866226092Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-05-13T20:47:32.866963089Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-13T20:47:32.867091247Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.75 inf=eth0
ts=2026-05-13T20:47:32.868946439Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:47:32.868965414Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-13T20:47:32.881949155Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-05-13T20:47:32.881961882Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-13T20:47:32.881969913Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-13T20:47:32.881980801Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-13T20:47:32.881973286Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-05-13T20:47:32.881975361Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-13T20:47:32.881990962Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-05-13T20:47:32.882043775Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-13T20:47:32.882071141Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-13T20:47:32.882080519Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-13T20:47:32.882073264Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-13T20:47:32.882462094Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-05-13T20:47:32.882465476Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-13T20:47:32.882470972Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-05-13T20:47:32.88247521Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-13T20:47:32.8825044Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-13T20:47:32.882532274Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-13T20:47:32.882536258Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-13T20:47:32.882539595Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-13T20:47:32.882651334Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-13T20:47:32.882656512Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-13T20:47:32.882660123Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-13T20:47:32.882663258Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-13T20:47:32.882665743Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-13T20:47:32.882668267Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-13T20:47:32.882686383Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-13T20:47:32.882693627Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T20:47:32.882734353Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=20.003µs success=true
level=info ts=2026-05-13T20:47:32.882747289Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-13T20:47:32.882794507Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=108.679µs errors=false
level=info ts=2026-05-13T20:47:32.882804847Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-13T20:47:32.882964086Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=279.016µs errors=false
level=info ts=2026-05-13T20:47:32.882970858Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-13T20:47:32.882976882Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=291.487µs
level=info ts=2026-05-13T20:47:32.883000642Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-13T20:47:32.883036167Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-13T20:47:32.883057742Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-13T20:47:32.883044427Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=1.373229587s
level=info ts=2026-05-13T20:47:32.883128584Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-13T20:47:32.883152507Z caller=loki.go:615 msg="Loki started" startup_time=90.16631ms
level=debug ts=2026-05-13T20:47:32.883387696Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-13T20:47:32.8833959Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-13T20:47:32.883442619Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-13T20:47:33.980903987Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:47:33.980926734Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:47:36.533437852Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:47:36.533461656Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T20:47:37.883509509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:47:42.79000954Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:47:42.790055881Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T20:47:42.883535478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:47:47.88349903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:47:48.515536077Z caller=logging.go:137 msg="GET /ready (503) 85.857µs"
level=warn ts=2026-05-13T20:47:52.88378281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:47:56.101407853Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.76:7946: dial tcp 10.129.0.76:7946: connect: connection refused"
ts=2026-05-13T20:47:56.102245466Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.77:7946: dial tcp 10.129.0.77:7946: connect: connection refused"
ts=2026-05-13T20:47:56.104215023Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.66:7946: dial tcp 10.130.0.66:7946: connect: connection refused"
ts=2026-05-13T20:47:56.105986384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.67:7946"
ts=2026-05-13T20:47:56.109109373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.68:7946"
ts=2026-05-13T20:47:56.111563387Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.69:7946"
level=info ts=2026-05-13T20:47:56.112921631Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=23.246688768s
level=warn ts=2026-05-13T20:47:57.883405466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:47:58.514638514Z caller=logging.go:137 msg="GET /ready (503) 75.044µs"
level=debug ts=2026-05-13T20:47:59.279653782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T20:48:02.883158584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T20:48:02.883202517Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-13T20:48:02.883211462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:48:02.883238076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:48:04.117389664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=warn ts=2026-05-13T20:48:07.883720096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:48:12.883895044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:48:17.883343592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:48:22.883876403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:48:27.883774858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:48:31.500922377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T20:48:32.864698873Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:48:32.864713893Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:48:32.884032128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:48:32.884048266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:48:32.884056198Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:48:32.88408199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:48:33.143446888Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T20:48:33.143503514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:48:33.15531006Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T20:48:34.121386328Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T20:48:35.622529096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39154"
level=warn ts=2026-05-13T20:48:37.883569809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:48:42.88406939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:48:47.884064366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:48:48.702278017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:48:48.708844938Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T20:48:48.900380355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T20:48:52.88402944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:48:57.884007004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:49:01.567114262Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T20:49:02.883928893Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:49:02.883955793Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:49:02.883963695Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:49:02.883973734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:49:04.124701137Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T20:49:05.625876813Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49478"
level=warn ts=2026-05-13T20:49:07.883877836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:49:07.980782987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40382"
level=warn ts=2026-05-13T20:49:12.883563009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:49:14.674914851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T20:49:17.8833594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:49:22.883369322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:49:23.774946335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:49:23.782010876Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T20:49:27.883576797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:49:31.132964859Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T20:49:32.86427196Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:49:32.865323244Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T20:49:32.883683942Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:49:32.883703838Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:49:32.88372112Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:49:32.883773611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:49:34.128104769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T20:49:35.628652983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40246"
level=warn ts=2026-05-13T20:49:37.883645363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:49:37.963619885Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T20:49:42.8840728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:49:45.789060474Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47798"
level=warn ts=2026-05-13T20:49:47.883407735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:49:52.883357671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:49:57.884047519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:50:01.39888329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T20:50:02.883473827Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:50:02.883503129Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:50:02.883511812Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:50:02.883519529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:50:04.131402403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=warn ts=2026-05-13T20:50:07.884009914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:50:10.639720062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T20:50:10.641181185Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:50:12.884082276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:50:17.883751593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:50:21.990978025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:50:22.000649987Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T20:50:22.883616165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:50:27.884260567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:50:31.51717698Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T20:50:32.865392945Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:50:32.865955162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:50:32.883173314Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:50:32.883190961Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:50:32.883198982Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:50:32.88320744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:50:34.134314112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=warn ts=2026-05-13T20:50:37.883839776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:50:42.883267855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:50:47.884129031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:50:52.129709935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-13T20:50:52.883196897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:50:57.883757433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:51:02.883317125Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:51:02.883348885Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:51:02.883356747Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:51:02.883345792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:51:04.13754648Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T20:51:06.782527363Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T20:51:07.884208154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:51:07.988691821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53650"
level=info ts=2026-05-13T20:51:10.762519162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:51:10.771479388Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T20:51:12.883830731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:51:17.883998494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:51:22.883574877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:51:27.883335603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:51:32.470503957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T20:51:32.864961535Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:51:32.865348092Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:51:32.883678508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:51:32.883695647Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:51:32.883703607Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:51:32.883721868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:51:34.140320783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=warn ts=2026-05-13T20:51:37.883729314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:51:42.883555074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:51:47.884064855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:51:52.883831567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:51:57.181882234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:51:57.188474583Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T20:51:57.884006935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:52:00.170444015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T20:52:02.883452826Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:52:02.883479716Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:52:02.883487496Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:52:02.883501648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:52:04.14284142Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T20:52:07.497929857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T20:52:07.883391805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:52:12.883289237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:52:17.883794738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:52:22.88339543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:52:27.883602215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:52:32.865518036Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:52:32.865927548Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:52:32.883128802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T20:52:32.883136998Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-13T20:52:32.883150147Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:52:32.883157974Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:52:32.883192681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:52:32.883268899Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-13T20:52:32.883277397Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T20:52:32.88329105Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=8.981µs success=true
level=debug ts=2026-05-13T20:52:32.883459431Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-13T20:52:34.146211416Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T20:52:37.170042415Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T20:52:37.170065924Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-13T20:52:37.883617607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:52:41.455556892Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=debug ts=2026-05-13T20:52:42.589681522Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T20:52:42.883897564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:52:45.498429046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:52:45.50492616Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T20:52:45.741607761Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
ts=2026-05-13T20:52:45.799340571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47458"
level=warn ts=2026-05-13T20:52:47.884249693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:52:50.027379353Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T20:52:52.883769364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:52:54.312437795Z caller=checkpoint.go:372 msg="writing series" size="367 kB"
level=warn ts=2026-05-13T20:52:57.883621218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:52:58.598875088Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-13T20:53:02.040754367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-13T20:53:02.884069091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:53:02.884095455Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:53:02.884104081Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:53:02.884121425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:02.88414441Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-13T20:53:02.884157993Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-13T20:53:04.149044958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T20:53:05.639347333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42524"
level=debug ts=2026-05-13T20:53:07.170112578Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-13T20:53:07.883367433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:10.766653621Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T20:53:11.45551661Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-13T20:53:12.883352071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:15.740719289Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T20:53:17.884070248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:20.026421219Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-13T20:53:22.883532873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:24.312287176Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-13T20:53:27.883646022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:28.598393926Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-13T20:53:32.865049091Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:53:32.865491287Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:53:32.883877747Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:53:32.883882047Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T20:53:32.883879396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:32.883892799Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:53:32.883900352Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T20:53:34.152486196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T20:53:35.64271094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47294"
level=debug ts=2026-05-13T20:53:37.16936251Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-05-13T20:53:37.883894704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:53:37.995999438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33346"
level=info ts=2026-05-13T20:53:39.988836408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:53:39.996944726Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T20:53:41.455007709Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=debug ts=2026-05-13T20:53:41.892578485Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T20:53:42.883744964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:45.741044386Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-13T20:53:45.803490346Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:45324"
level=warn ts=2026-05-13T20:53:47.883931791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:50.027190822Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-13T20:53:52.883442345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:54.312617448Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-13T20:53:57.884200483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:53:58.59808392Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-13T20:54:02.883934328Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:54:02.883944232Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-13T20:54:02.883966737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:02.883995754Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:54:02.884005183Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T20:54:04.155330293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T20:54:06.902902428Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T20:54:07.169664949Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T20:54:07.883897799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:07.943751246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-13T20:54:07.999055515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44344"
level=debug ts=2026-05-13T20:54:11.455620651Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T20:54:12.883838819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:15.74117944Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
ts=2026-05-13T20:54:15.806184221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:54972"
level=warn ts=2026-05-13T20:54:17.883638201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:20.026724643Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-13T20:54:22.883998972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:24.313073561Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-13T20:54:24.313096772Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=19
level=warn ts=2026-05-13T20:54:27.883762444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:28.598541746Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-13T20:54:32.864777675Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:54:32.865205838Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:54:32.883451595Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:54:32.883470093Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:54:32.883477444Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:54:32.883503195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:32.883516528Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T20:54:34.007400125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:54:34.015270546Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T20:54:34.158299906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T20:54:34.4511367Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T20:54:37.169645955Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-13T20:54:37.883934455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:41.45533378Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=warn ts=2026-05-13T20:54:42.883537181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:45.740907007Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
ts=2026-05-13T20:54:45.808786787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34834"
level=warn ts=2026-05-13T20:54:47.883389621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:50.026553763Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T20:54:52.884227798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:54.3123188Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-13T20:54:57.8835307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:54:58.598760453Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T20:54:59.290223553Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T20:55:02.883521316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:55:02.883536185Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:55:02.883554889Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:55:02.883562093Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T20:55:02.883743944Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
ts=2026-05-13T20:55:04.161495523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T20:55:07.169669447Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=warn ts=2026-05-13T20:55:07.883514806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:11.454957882Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-13T20:55:11.45498272Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-05-13T20:55:12.883588728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:15.741209376Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=debug ts=2026-05-13T20:55:15.876292781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-13T20:55:15.8780012Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T20:55:17.883840388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:20.027098236Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=warn ts=2026-05-13T20:55:22.883399043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:24.312561493Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-13T20:55:25.298273246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:55:25.305563814Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:55:25.305587384Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T20:55:25.305622408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:55:25.311915154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T20:55:27.883860669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:28.598130139Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-13T20:55:32.864464081Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:55:32.865513438Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T20:55:32.883903545Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:55:32.88392054Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:55:32.88392784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:55:32.883934163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:32.884002411Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
ts=2026-05-13T20:55:34.163883104Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T20:55:34.462803038Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T20:55:37.16941128Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-13T20:55:37.88402238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:55:38.165331633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:55:38.174895942Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T20:55:41.45528276Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-13T20:55:42.884007172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:45.741491956Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-13T20:55:45.812087956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34486"
level=warn ts=2026-05-13T20:55:47.883583384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:50.026986195Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T20:55:52.883237957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:54.317635024Z caller=checkpoint.go:372 msg="writing series" size="6.4 MB"
level=debug ts=2026-05-13T20:55:54.317657599Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.2 MB" series=10
level=warn ts=2026-05-13T20:55:57.883365798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:55:58.597793124Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-13T20:56:02.88389091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:56:02.883917786Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-13T20:56:02.883911704Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-13T20:56:02.88392698Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:56:02.883939174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:56:04.166914073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T20:56:05.651201222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56830"
level=debug ts=2026-05-13T20:56:07.169333039Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T20:56:07.883806788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:56:08.767818594Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T20:56:10.143608143Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T20:56:11.455215302Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-13T20:56:12.88343443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:56:13.694776338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:56:13.703412076Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T20:56:15.741002632Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-13T20:56:17.883452366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:56:20.027370858Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-13T20:56:22.88393899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:56:24.31287001Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-13T20:56:27.883384391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:56:28.598355273Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-13T20:56:32.865431836Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:56:32.865852647Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-13T20:56:32.883194481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:56:32.883203246Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:56:32.883219731Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:56:32.883227361Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T20:56:32.884368483Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
ts=2026-05-13T20:56:34.170159265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T20:56:35.148260341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T20:56:37.16994733Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-13T20:56:37.883957498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:56:41.45492109Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-13T20:56:42.883390519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:56:45.740804794Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
ts=2026-05-13T20:56:45.817134793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39958"
level=warn ts=2026-05-13T20:56:47.883739278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:56:50.026851642Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-13T20:56:52.883416987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:56:54.313093774Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T20:56:57.883607754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:56:58.598268586Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-13T20:57:02.884047954Z caller=checkpoint.go:384 msg="flushing series" totalSize="560 kB" series=15
level=info ts=2026-05-13T20:57:02.884059197Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:57:02.884083158Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:57:02.884090581Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:57:02.884091504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:57:02.889432597Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-13T20:57:02.893120777Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009707128s
ts=2026-05-13T20:57:04.17318581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T20:57:06.110902824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T20:57:07.883399707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:57:08.009131562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41070"
level=debug ts=2026-05-13T20:57:09.102739287Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T20:57:12.049138827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:57:12.057582024Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T20:57:12.883462172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:57:15.821032481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43406"
level=warn ts=2026-05-13T20:57:17.883229083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:57:22.883831442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T20:57:27.883198051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:57:32.865455305Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:57:32.866101235Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:57:32.883507406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T20:57:32.883526115Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-13T20:57:32.883537188Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:57:32.883546377Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:57:32.883566899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:57:32.883822712Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-13T20:57:32.883837368Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T20:57:32.88385995Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.779µs success=true
level=debug ts=2026-05-13T20:57:32.884068457Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-13T20:57:34.17644891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T20:57:37.104284803Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T20:57:37.104307375Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=warn ts=2026-05-13T20:57:37.883979982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:57:41.322524769Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
level=warn ts=2026-05-13T20:57:42.883952749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:57:44.789391279Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T20:57:45.54082968Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=warn ts=2026-05-13T20:57:47.883903307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:57:49.759380222Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-05-13T20:57:49.819010446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T20:57:52.883212479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:57:53.979748269Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-13T20:57:53.97977087Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=4
level=warn ts=2026-05-13T20:57:57.883218758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:57:58.197321905Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=debug ts=2026-05-13T20:58:02.41624676Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-13T20:58:02.883428535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T20:58:02.883437622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:58:02.883467586Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:58:02.883480452Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T20:58:04.179574149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T20:58:06.634638844Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-13T20:58:07.437748575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:58:07.448981536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T20:58:07.883739986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:58:08.013501079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37358"
level=debug ts=2026-05-13T20:58:10.853206668Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-13T20:58:12.883634644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:58:15.072058497Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-13T20:58:16.533272686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T20:58:16.533346615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T20:58:16.542227827Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T20:58:17.88382464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:58:19.290963771Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-13T20:58:22.883872776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:58:23.509686252Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-13T20:58:27.727902545Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T20:58:27.883813176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:58:29.171339495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T20:58:29.184462933Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T20:58:31.947564015Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T20:58:32.864632963Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:58:32.865407265Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T20:58:32.883910171Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:58:32.883933047Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:58:32.883942881Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:58:32.88396019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:58:34.183077783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T20:58:36.166390483Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-05-13T20:58:37.883829941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:58:40.38498793Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-13T20:58:42.883957632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:58:44.603330043Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-13T20:58:45.826410374Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59744"
level=debug ts=2026-05-13T20:58:46.232999032Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T20:58:47.883453363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:58:48.822114783Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-13T20:58:49.031933346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:58:49.039087394Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T20:58:51.078991716Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T20:58:52.883351669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:58:53.041077955Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-13T20:58:57.204074541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T20:58:57.211501822Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T20:58:57.259722833Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=warn ts=2026-05-13T20:58:57.884085633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:59:01.47844534Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-13T20:59:02.883390557Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:59:02.883420705Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:59:02.883431361Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T20:59:02.883433243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:59:04.186213949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T20:59:05.660888283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55442"
level=debug ts=2026-05-13T20:59:05.697526848Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-13T20:59:07.883198224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:59:09.915853899Z caller=checkpoint.go:372 msg="writing series" size="299 kB"
level=debug ts=2026-05-13T20:59:09.915874792Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=18
level=warn ts=2026-05-13T20:59:12.883367885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:59:14.134959188Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
ts=2026-05-13T20:59:15.8295947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56920"
level=warn ts=2026-05-13T20:59:17.884186247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:59:18.353182481Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-13T20:59:18.61048674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T20:59:22.571879121Z caller=checkpoint.go:372 msg="writing series" size="366 kB"
level=warn ts=2026-05-13T20:59:22.884121565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:59:26.790699596Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=warn ts=2026-05-13T20:59:27.883850489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:59:31.009561337Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T20:59:32.865114445Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T20:59:32.86600892Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-13T20:59:32.88335939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T20:59:32.883369858Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T20:59:32.883394653Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T20:59:32.883413064Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T20:59:34.18935358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T20:59:35.228717968Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-13T20:59:35.665116941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55904"
level=warn ts=2026-05-13T20:59:37.883717674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T20:59:38.020705852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:42000"
level=debug ts=2026-05-13T20:59:39.447704056Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-13T20:59:39.447738459Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-13T20:59:41.122428928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T20:59:41.136090522Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T20:59:42.883363927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:59:43.666329211Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=debug ts=2026-05-13T20:59:44.722101475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T20:59:44.724303635Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T20:59:44.772555267Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T20:59:45.833370716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34078"
level=info ts=2026-05-13T20:59:47.701832563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T20:59:47.709621693Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T20:59:47.883774242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:59:47.884864895Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-13T20:59:52.103225728Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-13T20:59:52.883979812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T20:59:56.32175766Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T20:59:57.883811554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:00:00.540966696Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-13T21:00:02.883824414Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:00:02.883854721Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:00:02.883862494Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:00:02.883858416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:00:04.19281361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:00:04.759385635Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
ts=2026-05-13T21:00:05.668259151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56402"
level=warn ts=2026-05-13T21:00:07.884189934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:00:08.02413336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39122"
level=debug ts=2026-05-13T21:00:08.97797454Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=debug ts=2026-05-13T21:00:09.193253462Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:00:12.884072937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:00:13.196962769Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-13T21:00:15.835972084Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46232"
level=debug ts=2026-05-13T21:00:17.416315743Z caller=checkpoint.go:372 msg="writing series" size="472 kB"
level=debug ts=2026-05-13T21:00:17.416340835Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
level=warn ts=2026-05-13T21:00:17.884065554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:00:21.634911226Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-13T21:00:22.884049231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:00:25.853700319Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-13T21:00:27.884009227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:00:29.815216527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:00:30.072397968Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=info ts=2026-05-13T21:00:32.864676776Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:00:32.865263557Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=debug ts=2026-05-13T21:00:32.866511701Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T21:00:32.866555045Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976339 n=1
level=debug ts=2026-05-13T21:00:32.866596007Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976339 n=1
level=info ts=2026-05-13T21:00:32.883778668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:00:32.883797162Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:00:32.883806602Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:00:32.883846573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:00:34.196224436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:00:34.290834457Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-13T21:00:37.884078382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:00:38.509718375Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-13T21:00:39.163071444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:00:39.170852428Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:00:40.655806477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:00:40.665138165Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:00:41.89386959Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:00:42.728169713Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T21:00:42.883339287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:00:45.839006208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46364"
level=debug ts=2026-05-13T21:00:46.947457042Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T21:00:47.883395848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:00:51.170205504Z caller=checkpoint.go:372 msg="writing series" size="6.7 MB"
level=debug ts=2026-05-13T21:00:51.170226879Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.2 MB" series=8
level=warn ts=2026-05-13T21:00:52.883356973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:00:55.384706016Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-13T21:00:57.883568043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:00:59.603151528Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-13T21:01:02.883685092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:01:02.883712823Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:01:02.883720561Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:01:02.88373539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:03.821598524Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-13T21:01:04.199578418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:01:05.673603165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58432"
level=warn ts=2026-05-13T21:01:07.884072808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:08.040404874Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=debug ts=2026-05-13T21:01:12.259286515Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T21:01:12.88384319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:13.003253803Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:01:16.478310294Z caller=checkpoint.go:372 msg="writing series" size="475 kB"
level=warn ts=2026-05-13T21:01:17.884426161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:19.946715818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:01:20.697370596Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-13T21:01:22.883355739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:01:23.602078507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:01:23.612563339Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:01:24.916090888Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-13T21:01:27.883366125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:29.134857924Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-13T21:01:32.864622508Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:01:32.864675808Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:01:32.883187884Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:01:32.883215464Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:01:32.883225904Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:01:32.88322797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:33.35344085Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T21:01:33.353462083Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
ts=2026-05-13T21:01:34.203006971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:01:35.676571846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55572"
level=debug ts=2026-05-13T21:01:37.57214985Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-13T21:01:37.883908935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:01:38.030573871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50748"
level=info ts=2026-05-13T21:01:38.396100018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:01:38.4057192Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:01:41.791089968Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-13T21:01:42.883226456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:45.329900564Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:01:46.009870855Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-13T21:01:47.883232163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:50.228041943Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-13T21:01:52.883515816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:54.447530784Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T21:01:57.884007563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:01:58.666068219Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-13T21:02:02.884132033Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:02:02.884137878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:02:02.884137877Z caller=checkpoint.go:384 msg="flushing series" totalSize="270 kB" series=6
level=debug ts=2026-05-13T21:02:02.884163519Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:02:02.884172363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:02:03.140120878Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-13T21:02:03.142306958Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.258296812s
ts=2026-05-13T21:02:04.206140742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:02:05.679717443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59418"
level=warn ts=2026-05-13T21:02:07.883348222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:02:08.033549025Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51662"
level=info ts=2026-05-13T21:02:12.112276691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:02:12.120927853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:02:12.883852462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:02:15.849229833Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:34192"
level=warn ts=2026-05-13T21:02:17.883621996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:02:18.554748609Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:02:21.397526828Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:02:22.884165591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:02:27.883371762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:02:28.983594234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:02:28.991966704Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:02:28.991991522Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:02:28.992031892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:02:28.998708311Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:02:32.864895078Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:02:32.865326999Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:02:32.883833235Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T21:02:32.883850483Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:02:32.883869214Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:02:32.883877358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:02:32.883875196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:02:32.883938339Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-13T21:02:32.883945777Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:02:32.883961769Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.742µs success=true
level=debug ts=2026-05-13T21:02:32.88415422Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-13T21:02:34.209405752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:02:36.260748595Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T21:02:36.260770867Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
level=warn ts=2026-05-13T21:02:37.883380873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:02:38.036549891Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45604"
level=debug ts=2026-05-13T21:02:39.634558376Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
level=warn ts=2026-05-13T21:02:42.88372715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:02:43.010248853Z caller=checkpoint.go:372 msg="writing series" size="366 kB"
ts=2026-05-13T21:02:45.851783919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:44834"
level=debug ts=2026-05-13T21:02:46.3850877Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-13T21:02:47.883996611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:02:48.241980175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:02:48.250466273Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:02:49.771491532Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-13T21:02:49.771513134Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=4
level=debug ts=2026-05-13T21:02:52.459384711Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:02:52.884144182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:02:53.134841882Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-13T21:02:56.509642615Z caller=checkpoint.go:372 msg="writing series" size="437 kB"
level=info ts=2026-05-13T21:02:57.044063941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:02:57.05160084Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:02:57.595862991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:02:57.884044924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:02:59.88504029Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-13T21:03:02.883509064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:03:02.88353672Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:03:02.883546295Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:03:02.883540226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:03.259379274Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
ts=2026-05-13T21:03:04.212653389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:03:06.634373492Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-13T21:03:07.883876163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:10.009931752Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T21:03:12.883541898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:13.385164671Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=info ts=2026-05-13T21:03:14.105058184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:03:14.115463305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:03:16.760174471Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-13T21:03:17.883981563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:20.134505359Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-13T21:03:20.482710764Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:03:22.884045208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:23.509940098Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-13T21:03:26.88436307Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-05-13T21:03:27.883359586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:30.260159689Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-13T21:03:30.260182019Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-05-13T21:03:32.865662324Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:03:32.866244363Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:03:32.883550573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:03:32.883566515Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:03:32.883574261Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:03:32.883578593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:33.63461073Z caller=checkpoint.go:372 msg="writing series" size="415 kB"
ts=2026-05-13T21:03:34.215511483Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:03:35.685322364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53016"
level=debug ts=2026-05-13T21:03:37.009464843Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-13T21:03:37.883700623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:03:38.796830738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:03:38.814573814Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:03:40.3844526Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-13T21:03:42.883886105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:43.759889212Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-13T21:03:46.104868751Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:03:47.135212805Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=warn ts=2026-05-13T21:03:47.884009832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:50.509846478Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-13T21:03:52.883408488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:03:53.884345134Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=debug ts=2026-05-13T21:03:56.050473924Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:03:56.053258511Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T21:03:57.259821975Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-13T21:03:57.883363581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:00.635207652Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-13T21:04:02.883319736Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:04:02.88333524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:02.883353161Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:04:02.88336419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:04:04.009937033Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-13T21:04:04.218833447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:04:07.385258623Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=warn ts=2026-05-13T21:04:07.884132469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:10.759537738Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-13T21:04:10.759558138Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-05-13T21:04:12.884075213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:04:12.934228314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:04:12.944265039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:04:14.134460436Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T21:04:16.036083886Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:04:17.509313108Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=warn ts=2026-05-13T21:04:17.883350739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:20.884875216Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-13T21:04:22.884043583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:24.259271683Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=info ts=2026-05-13T21:04:25.621655076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="141 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:04:25.633106412Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:04:25.633140963Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:04:25.633179123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="141 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:04:25.64093315Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:04:27.634640836Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:04:27.883998318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:31.010279838Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:04:32.864661879Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:04:32.864982385Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:04:32.8835491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:04:32.883575114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:32.883585335Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:04:32.88359974Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:04:34.221580524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:04:34.384553455Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-13T21:04:37.759901908Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-13T21:04:37.883873796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:41.135295255Z caller=checkpoint.go:372 msg="writing series" size="275 kB"
level=warn ts=2026-05-13T21:04:42.883983885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:04:43.23606344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="141 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:04:43.248428896Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:04:44.509393319Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-13T21:04:44.509420028Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-13T21:04:47.883777366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:47.884810387Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-13T21:04:48.735361154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:04:51.259769075Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-13T21:04:52.883382512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:54.634383998Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T21:04:55.996736949Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:04:57.883529796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:04:58.010032505Z caller=checkpoint.go:372 msg="writing series" size="365 kB"
level=debug ts=2026-05-13T21:05:01.384631565Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=info ts=2026-05-13T21:05:02.104831709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:05:02.112552349Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:05:02.883885939Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:05:02.883916424Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:05:02.883928199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:05:02.883945154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:05:04.225104392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:05:04.75966425Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-13T21:05:07.884134783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:05:08.044725469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48036"
level=debug ts=2026-05-13T21:05:08.135206769Z caller=checkpoint.go:372 msg="writing series" size="414 kB"
level=debug ts=2026-05-13T21:05:08.135230495Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-05-13T21:05:11.509294695Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T21:05:12.883814711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:05:14.884825487Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-13T21:05:17.883772497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:05:18.260041394Z caller=checkpoint.go:372 msg="writing series" size="431 kB"
level=info ts=2026-05-13T21:05:20.395986309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="141 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:05:20.406131968Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:05:20.87090433Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:05:21.635195741Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-13T21:05:22.883568104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:05:25.009367008Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-13T21:05:27.884064169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:05:28.384831434Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-13T21:05:31.759510396Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=info ts=2026-05-13T21:05:32.865482864Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:05:32.865970143Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:05:32.883475326Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:05:32.883494874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:05:32.883503236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:05:32.883574779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:05:34.227544826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:05:35.134197739Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
ts=2026-05-13T21:05:35.692271931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:35930"
level=debug ts=2026-05-13T21:05:36.725431308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:05:37.8842192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:05:38.509701326Z caller=checkpoint.go:372 msg="writing series" size="431 kB"
level=debug ts=2026-05-13T21:05:38.509729745Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-13T21:05:41.884784163Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=warn ts=2026-05-13T21:05:42.884037373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:05:45.260100893Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=warn ts=2026-05-13T21:05:47.884124809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:05:48.634916201Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-13T21:05:50.420927775Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:05:52.009492334Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T21:05:52.883216501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:05:55.388668104Z caller=checkpoint.go:372 msg="writing series" size="7.7 MB"
level=debug ts=2026-05-13T21:05:55.388689778Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.0 MB" series=5
level=warn ts=2026-05-13T21:05:57.883536288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:05:58.760081392Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-13T21:05:58.86151662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:05:58.871515328Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:06:02.135085026Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-13T21:06:02.883755258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:06:02.883750708Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:06:02.883789716Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:06:02.88379934Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:06:03.850874862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="141 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:06:03.858372383Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T21:06:04.23076306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:06:05.510010221Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-13T21:06:05.697654892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48190"
level=warn ts=2026-05-13T21:06:07.883204802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:06:08.884154036Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=debug ts=2026-05-13T21:06:12.259907824Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-13T21:06:12.884001958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:06:15.634980702Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T21:06:17.883769367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:06:19.0104978Z caller=checkpoint.go:372 msg="writing series" size="682 kB"
level=debug ts=2026-05-13T21:06:20.320087014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:06:22.384729786Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T21:06:22.883224579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:06:25.759426722Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-13T21:06:26.2114362Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:06:27.883519116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:06:29.134782717Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=debug ts=2026-05-13T21:06:29.134810262Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=debug ts=2026-05-13T21:06:32.509489613Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=info ts=2026-05-13T21:06:32.864955282Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:06:32.865424762Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:06:32.884048188Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:06:32.884067353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:06:32.8840757Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:06:32.884090316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:06:34.233565776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:06:35.700799568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41626"
level=debug ts=2026-05-13T21:06:35.885282909Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=warn ts=2026-05-13T21:06:37.883899648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:06:38.050494256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39006"
level=debug ts=2026-05-13T21:06:39.259458344Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-13T21:06:42.634683893Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-13T21:06:42.883385492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:06:45.86489444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39176"
level=debug ts=2026-05-13T21:06:46.009888042Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-13T21:06:47.267582084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:06:47.279203184Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:06:47.883601865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:06:49.384457888Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-13T21:06:52.759881552Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T21:06:52.883979138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:06:55.434229452Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:06:56.135267194Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-13T21:06:56.514190487Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:06:57.883752101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:06:58.181534949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="165 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:06:58.190060208Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:06:59.509275691Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-13T21:07:02.883215128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:07:02.883213551Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:07:02.883243149Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:07:02.883252538Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:07:02.88430094Z caller=checkpoint.go:384 msg="flushing series" totalSize="798 kB" series=9
level=info ts=2026-05-13T21:07:02.893837911Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-13T21:07:02.898720985Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.014619233s
ts=2026-05-13T21:07:04.236700776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:07:05.704223297Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51106"
level=warn ts=2026-05-13T21:07:07.883387068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:07:08.05309205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35154"
level=warn ts=2026-05-13T21:07:12.883460036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:07:17.883377065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:07:22.884308434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:07:23.125725297Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:07:27.883688249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:07:32.733500988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.9 MB" avg_comp="2.5 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:07:32.745511132Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:07:32.864697304Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:07:32.86520795Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:07:32.883787611Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:07:32.883812469Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:07:32.883823269Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:07:32.88382638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:07:32.883833982Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T21:07:32.884155655Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-13T21:07:32.884167239Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:07:32.884193355Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.017µs success=true
level=debug ts=2026-05-13T21:07:32.884405665Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-13T21:07:34.239463123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:07:36.261456237Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T21:07:36.261485688Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=warn ts=2026-05-13T21:07:37.883257302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:07:39.635417636Z caller=checkpoint.go:372 msg="writing series" size="399 kB"
level=warn ts=2026-05-13T21:07:42.883196804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:07:43.009982137Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-13T21:07:46.384976149Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-05-13T21:07:46.38499949Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-05-13T21:07:47.883697493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:07:49.784468947Z caller=checkpoint.go:372 msg="writing series" size="23 MB"
level=debug ts=2026-05-13T21:07:49.784520504Z caller=checkpoint.go:384 msg="flushing series" totalSize="23 MB" series=1
level=warn ts=2026-05-13T21:07:52.883506941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:07:53.135504828Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-13T21:07:53.220533555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:07:53.222491889Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T21:07:53.245585732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="188 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:07:53.253461518Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:07:55.851622884Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:07:56.510289662Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=warn ts=2026-05-13T21:07:57.884255527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:07:59.885423679Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-05-13T21:08:02.883674762Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:08:02.88367815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:02.883702737Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:08:02.883712166Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:08:03.259570256Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
ts=2026-05-13T21:08:04.242966506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:08:06.634889044Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-13T21:08:06.634914048Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-13T21:08:07.883586935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:08:08.057805407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46998"
level=debug ts=2026-05-13T21:08:10.010134622Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T21:08:12.883909065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:13.384674304Z caller=checkpoint.go:372 msg="writing series" size="6.4 kB"
ts=2026-05-13T21:08:15.870629581Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39772"
level=debug ts=2026-05-13T21:08:16.76047319Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-13T21:08:17.883358746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:20.134921381Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-13T21:08:20.375765194Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:08:22.884099317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:23.50987905Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T21:08:23.730388834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:08:23.740986876Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:08:26.88490932Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-05-13T21:08:27.883856784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:30.260576291Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=info ts=2026-05-13T21:08:32.864775858Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:08:32.86518722Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:08:32.883743454Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:08:32.883766224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:08:32.883775994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:08:32.883789417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:33.635136899Z caller=checkpoint.go:372 msg="writing series" size="807 kB"
level=debug ts=2026-05-13T21:08:33.635160266Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
ts=2026-05-13T21:08:34.246660463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T21:08:35.263219302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="188 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:08:35.272290739Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T21:08:35.710127228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56326"
level=debug ts=2026-05-13T21:08:37.009974865Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-13T21:08:37.884036642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:40.384890722Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-13T21:08:42.88364625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:43.760445319Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-13T21:08:47.134426788Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=warn ts=2026-05-13T21:08:47.88386752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:50.510368098Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-13T21:08:52.88349709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:08:53.88548021Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=debug ts=2026-05-13T21:08:54.382203756Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:08:54.395432291Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:08:57.259895168Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=warn ts=2026-05-13T21:08:57.884010294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:00.635110598Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-13T21:09:02.883633439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:09:02.883671275Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:09:02.883679309Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:09:02.883675634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:04.009859046Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-13T21:09:04.249763659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:09:05.713182896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47998"
level=debug ts=2026-05-13T21:09:07.384778629Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=warn ts=2026-05-13T21:09:07.883813049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:10.760518816Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-13T21:09:12.884135987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:14.134503237Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T21:09:14.270450332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:09:14.278834201Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:09:14.27885831Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:09:14.278883261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:09:14.285789588Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T21:09:15.874176843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:42964"
level=debug ts=2026-05-13T21:09:17.510326804Z caller=checkpoint.go:372 msg="writing series" size="419 kB"
level=debug ts=2026-05-13T21:09:17.510348973Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=13
level=warn ts=2026-05-13T21:09:17.883804201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:20.885128522Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:09:21.761465601Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:09:22.883392011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:24.25957292Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=info ts=2026-05-13T21:09:25.026794592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:09:25.037688269Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:09:27.63728055Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-13T21:09:27.637303536Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=warn ts=2026-05-13T21:09:27.883346614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:09:30.798263128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="212 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:09:30.81175466Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:09:31.009718495Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:09:32.864786531Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:09:32.865141806Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:09:32.883586517Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:09:32.883611224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:09:32.8836215Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:09:32.883643254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:09:34.25454591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:09:34.384799773Z caller=checkpoint.go:372 msg="writing series" size="344 kB"
level=debug ts=2026-05-13T21:09:37.759526814Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-05-13T21:09:37.883699627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:09:38.063613807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60662"
level=debug ts=2026-05-13T21:09:39.981683649Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:09:41.135094317Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-13T21:09:42.883830645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:44.509495299Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-13T21:09:47.883607397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:47.884737526Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=info ts=2026-05-13T21:09:49.501573789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:09:49.515487364Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:09:51.260244251Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-13T21:09:51.692236471Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:09:52.884078539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:54.635472081Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T21:09:57.883602189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:09:58.010248761Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
level=debug ts=2026-05-13T21:09:58.010275211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-05-13T21:10:01.384504541Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T21:10:02.883952982Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:10:02.88398444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:10:02.883995357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:10:02.884010027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:10:04.257843839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:10:04.760248265Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-13T21:10:07.883833866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:10:08.135270619Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-13T21:10:11.509939835Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T21:10:12.884167878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:10:14.884903021Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-13T21:10:17.173171608Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:10:17.883935993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:10:18.260200449Z caller=checkpoint.go:372 msg="writing series" size="418 kB"
level=debug ts=2026-05-13T21:10:18.260220387Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=debug ts=2026-05-13T21:10:21.635104709Z caller=checkpoint.go:372 msg="writing series" size="506 kB"
level=warn ts=2026-05-13T21:10:22.883497328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:10:24.806787217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="235 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:10:24.816383976Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:10:25.010323129Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=debug ts=2026-05-13T21:10:25.514995471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:10:27.883501246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:10:28.385152576Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T21:10:29.695462102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:10:29.703114637Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:10:31.760139187Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-13T21:10:32.864633324Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:10:32.865377054Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:10:32.883897241Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:10:32.883924106Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:10:32.883932598Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:10:32.883930291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:10:34.260815013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:10:35.135116929Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:10:37.88400513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:10:38.068043777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50008"
level=debug ts=2026-05-13T21:10:38.509721063Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=debug ts=2026-05-13T21:10:38.509743377Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=debug ts=2026-05-13T21:10:41.885254598Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=warn ts=2026-05-13T21:10:42.883653557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:10:45.260266787Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
ts=2026-05-13T21:10:45.879765457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51544"
level=warn ts=2026-05-13T21:10:47.884067613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:10:48.634612384Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-13T21:10:51.463491072Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:10:52.010234623Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T21:10:52.88321926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:10:55.397817561Z caller=checkpoint.go:372 msg="writing series" size="8.4 MB"
level=debug ts=2026-05-13T21:10:55.397848005Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.0 MB" series=5
level=warn ts=2026-05-13T21:10:57.883857867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:10:58.76026533Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-13T21:11:02.134767537Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-13T21:11:02.883618968Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:11:02.883652979Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:11:02.883663508Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:11:02.88365164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:11:04.267169718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:11:05.510318769Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T21:11:07.883837733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:11:08.885240058Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=debug ts=2026-05-13T21:11:08.994418121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:11:12.260127326Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-13T21:11:12.883577798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:11:13.192323038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.5 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:11:13.201226341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:11:15.251265143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:11:15.259307501Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:11:15.259327888Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:11:15.259357614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:11:15.266958734Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:11:15.634745712Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T21:11:17.884005744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:11:19.010434908Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T21:11:19.010457762Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-05-13T21:11:22.385242753Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-13T21:11:22.883736482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:11:25.760164102Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-13T21:11:26.776278889Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:11:27.883805969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:11:29.135606076Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=info ts=2026-05-13T21:11:31.750075022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:11:31.762157047Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:11:32.510598791Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=info ts=2026-05-13T21:11:32.864678443Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:11:32.865325724Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:11:32.883818186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:11:32.883839705Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:11:32.883850005Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:11:32.883905885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:11:34.270801353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:11:35.885243403Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-13T21:11:37.883483414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:11:39.259757066Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-13T21:11:42.635074951Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-13T21:11:42.883766059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:11:46.009967134Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-13T21:11:47.883221971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:11:49.385106998Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-13T21:11:52.760079933Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=warn ts=2026-05-13T21:11:52.883267496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:11:56.135100305Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-13T21:11:57.168271027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:11:57.176147015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:11:57.883974343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:11:58.619765402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:11:58.621583798Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T21:11:59.510549118Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
level=debug ts=2026-05-13T21:11:59.510575524Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=debug ts=2026-05-13T21:12:00.872256896Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:12:02.883410806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:12:02.883431183Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:12:02.883454647Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:12:02.883466796Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:12:02.89019031Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-13T21:12:02.892425416Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008077004s
level=info ts=2026-05-13T21:12:04.163798093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:12:04.175711474Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T21:12:04.273872461Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=warn ts=2026-05-13T21:12:07.88388194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:12:12.883751126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:12:15.885574916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52408"
level=warn ts=2026-05-13T21:12:17.883236021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:12:22.883235167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:12:27.883418881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:12:32.864928638Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:12:32.865443512Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:12:32.883872177Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T21:12:32.883969558Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:12:32.883970298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:12:32.883988238Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:12:32.883988954Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-13T21:12:32.883996437Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:12:32.883996701Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:12:32.884028921Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=27.642µs success=true
level=debug ts=2026-05-13T21:12:32.884212195Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T21:12:34.277107165Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:12:34.869327925Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:12:36.17935527Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-13T21:12:36.179378047Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=2
level=warn ts=2026-05-13T21:12:37.884211811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:12:39.47052312Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-13T21:12:42.762616489Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=warn ts=2026-05-13T21:12:42.88384377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:12:46.056450036Z caller=checkpoint.go:372 msg="writing series" size="953 kB"
level=debug ts=2026-05-13T21:12:46.056472355Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-13T21:12:46.232573042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:12:46.242894273Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:12:47.883506931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:12:49.348574349Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=info ts=2026-05-13T21:12:51.496097458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:12:51.503404354Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:12:52.674150827Z caller=checkpoint.go:372 msg="writing series" size="38 MB"
level=debug ts=2026-05-13T21:12:52.674191841Z caller=checkpoint.go:384 msg="flushing series" totalSize="38 MB" series=2
level=warn ts=2026-05-13T21:12:52.883240561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:12:53.39216321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:12:55.933341016Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-13T21:12:57.884048897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:12:59.227118662Z caller=checkpoint.go:372 msg="writing series" size="918 kB"
level=debug ts=2026-05-13T21:12:59.227143813Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-05-13T21:13:02.519386485Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-05-13T21:13:02.883366924Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:13:02.883396052Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:13:02.883406246Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:13:02.883400046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:13:04.280326545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:13:05.811638185Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-13T21:13:07.884063197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:13:09.104537581Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-13T21:13:09.929224309Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:13:12.397091279Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T21:13:12.883390587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:13:15.689476654Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-13T21:13:17.883910265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:13:18.982554947Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-13T21:13:22.275518694Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T21:13:22.883730641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:13:25.56813021Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T21:13:27.88378863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:13:28.860295487Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-13T21:13:29.134240875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:13:32.153342691Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-13T21:13:32.864695979Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:13:32.865279235Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:13:32.883786523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:13:32.883806899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:13:32.883816627Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:13:32.883838802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:13:34.283094826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:13:34.772237351Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:13:35.447363162Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T21:13:35.447385784Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=11
level=warn ts=2026-05-13T21:13:37.883253256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:13:38.739050611Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-13T21:13:39.398071958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:13:39.405263548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:13:42.030941923Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-13T21:13:42.883246217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:13:45.323506591Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-13T21:13:45.89091529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35808"
level=warn ts=2026-05-13T21:13:47.883773881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:13:48.469913973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="306 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:13:48.479541466Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:13:48.616547546Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-05-13T21:13:51.90926393Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-13T21:13:52.883855485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:13:55.201998856Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=warn ts=2026-05-13T21:13:57.883361863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:13:58.49731117Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=debug ts=2026-05-13T21:14:01.787695553Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-13T21:14:02.883127401Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:14:02.883156321Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:14:02.883168037Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:14:02.883206281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:14:04.286256468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:14:05.080086305Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-13T21:14:07.884231957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:14:08.37260394Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-13T21:14:10.356214643Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:14:11.665720175Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-13T21:14:11.665745593Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-05-13T21:14:12.883492819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:14:14.441865854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:14:14.957678237Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T21:14:17.883627718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:14:18.250675382Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
level=debug ts=2026-05-13T21:14:21.543662043Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-13T21:14:22.88393821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:14:24.836031531Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-13T21:14:27.883370627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:14:28.130980614Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:14:28.13100279Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=5
level=info ts=2026-05-13T21:14:30.195115172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:14:30.208045471Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:14:31.421537266Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:14:32.864646539Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:14:32.864846688Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:14:32.883238466Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:14:32.883256276Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:14:32.883263967Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:14:32.883264696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:14:34.289102003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:14:34.713836559Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-13T21:14:35.687499966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:14:35.699452758Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:14:37.883730846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:14:38.007094066Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-13T21:14:40.999106789Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:14:41.302227Z caller=checkpoint.go:372 msg="writing series" size="321 kB"
level=warn ts=2026-05-13T21:14:42.884080573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:14:44.598173449Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
ts=2026-05-13T21:14:45.89718584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:51342"
level=warn ts=2026-05-13T21:14:47.883866719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:14:47.884971648Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-13T21:14:50.672526037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:14:51.177062977Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T21:14:52.883996139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:14:54.469660484Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T21:14:57.763308283Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-13T21:14:57.763330764Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
level=warn ts=2026-05-13T21:14:57.883928455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:01.055838134Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=info ts=2026-05-13T21:15:02.883365708Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:15:02.883399484Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:15:02.883410438Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:15:02.883414982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:15:04.291678455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:15:04.348224499Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-13T21:15:05.729691867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43004"
level=debug ts=2026-05-13T21:15:07.640761717Z caller=checkpoint.go:372 msg="writing series" size="509 kB"
level=warn ts=2026-05-13T21:15:07.883939548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:10.933618991Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-13T21:15:12.077299267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:15:12.077355971Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T21:15:12.085477571Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:15:12.883314591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:14.22582031Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=info ts=2026-05-13T21:15:16.196196465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:15:16.204552313Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:15:16.204577464Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:15:16.204606152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:15:16.211905393Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:15:17.518539787Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:15:17.884058322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:20.811594639Z caller=checkpoint.go:372 msg="writing series" size="479 kB"
level=debug ts=2026-05-13T21:15:20.811620114Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=warn ts=2026-05-13T21:15:22.883852667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:24.104642498Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=info ts=2026-05-13T21:15:25.933303281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:15:25.943689425Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:15:27.397322088Z caller=checkpoint.go:372 msg="writing series" size="844 kB"
level=debug ts=2026-05-13T21:15:27.397345258Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-13T21:15:27.883995189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:15:28.235792836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:15:28.243793838Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:15:28.390584119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:15:28.397623885Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:15:30.689781674Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T21:15:32.864718953Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:15:32.86520431Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=debug ts=2026-05-13T21:15:32.866480092Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T21:15:32.866520182Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976340 n=1
level=debug ts=2026-05-13T21:15:32.866562194Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976340 n=1
level=info ts=2026-05-13T21:15:32.883669585Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:15:32.883687839Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:15:32.883697014Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:15:32.883722794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:33.982267873Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
ts=2026-05-13T21:15:34.294140192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:15:37.275474932Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-13T21:15:37.883975905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:40.208118837Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:15:40.210325565Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T21:15:40.567622881Z caller=checkpoint.go:372 msg="writing series" size="728 kB"
level=warn ts=2026-05-13T21:15:42.88389092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:43.860283607Z caller=checkpoint.go:372 msg="writing series" size="366 kB"
level=debug ts=2026-05-13T21:15:43.860307359Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
ts=2026-05-13T21:15:45.901796575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40160"
level=info ts=2026-05-13T21:15:46.011145536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:15:46.019776259Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:15:46.594212963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:15:47.153011192Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-05-13T21:15:47.883537046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:50.446082182Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T21:15:52.883592856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:15:53.738698226Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T21:15:56.163072441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:15:56.17233865Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:15:57.036710689Z caller=checkpoint.go:372 msg="writing series" size="9.1 MB"
level=debug ts=2026-05-13T21:15:57.036737155Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.2 MB" series=4
level=warn ts=2026-05-13T21:15:57.884096239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:00.323786628Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-13T21:16:02.88347716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:16:02.883519379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:16:02.883546426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:16:02.883557939Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:16:03.617047812Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
ts=2026-05-13T21:16:04.297549559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:16:06.908967953Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T21:16:07.883466246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:16:08.091825434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39878"
level=debug ts=2026-05-13T21:16:10.202130047Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-13T21:16:12.883248469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:13.160036047Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:16:13.494342784Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=info ts=2026-05-13T21:16:15.108723869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:16:15.119636074Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:16:16.78753897Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T21:16:17.883941201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:20.081793858Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T21:16:20.081821968Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=7
level=debug ts=2026-05-13T21:16:21.321422415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:16:22.883374908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:23.372324794Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:16:26.664938412Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-13T21:16:27.883737309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:16:28.948552573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:16:28.95846017Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:16:29.957799172Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=info ts=2026-05-13T21:16:32.864791856Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:16:32.864796321Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:16:32.883189807Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:16:32.883212982Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:16:32.883223992Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:16:32.883241498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:33.251165702Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
ts=2026-05-13T21:16:34.300404491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:16:36.543773925Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=warn ts=2026-05-13T21:16:37.884210658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:39.836171088Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=info ts=2026-05-13T21:16:40.721999042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:16:40.732220909Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:16:41.940911072Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:16:42.883535115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:43.128868889Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-13T21:16:46.421165522Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-13T21:16:47.883938261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:49.714078157Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-13T21:16:52.883488511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:53.006815858Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-13T21:16:56.299885301Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-13T21:16:57.883861262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:16:59.5929676Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-13T21:16:59.592990131Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=12
level=debug ts=2026-05-13T21:17:01.045622992Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:17:02.883381301Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:17:02.883412041Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:17:02.883420294Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:17:02.883414363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:17:04.303418624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T21:17:04.40712782Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-13T21:17:04.409589676Z caller=checkpoint.go:670 msg="checkpoint done" time=4m31.525431535s
level=warn ts=2026-05-13T21:17:07.883361187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:17:08.096658365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46240"
level=info ts=2026-05-13T21:17:08.951391806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:17:08.961119772Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:17:10.906216848Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:17:12.883893152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:17:15.907737433Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:58116"
level=warn ts=2026-05-13T21:17:17.883235087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:17:22.650142369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:17:22.661712316Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:17:22.884091517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:17:27.184957037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:17:27.192483599Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:17:27.883776057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:17:32.864411821Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:17:32.8654773Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T21:17:32.883994509Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T21:17:32.884043038Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:17:32.884064045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:17:32.884077851Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:17:32.884081114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:17:32.884123563Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-13T21:17:32.884132087Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:17:32.884149153Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.514µs success=true
level=debug ts=2026-05-13T21:17:32.884365253Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T21:17:34.306944382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:17:36.179936786Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-13T21:17:36.179960076Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=2
level=warn ts=2026-05-13T21:17:37.883921035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:17:38.333743311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:17:39.470053484Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:17:42.763218517Z caller=checkpoint.go:372 msg="writing series" size="438 kB"
level=warn ts=2026-05-13T21:17:42.884084017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:17:46.05614342Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T21:17:46.056165324Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-05-13T21:17:47.069543308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:17:47.883953755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:17:49.348878476Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-13T21:17:52.679362463Z caller=checkpoint.go:372 msg="writing series" size="44 MB"
level=debug ts=2026-05-13T21:17:52.679389601Z caller=checkpoint.go:384 msg="flushing series" totalSize="44 MB" series=2
level=warn ts=2026-05-13T21:17:52.883286738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:17:55.933891626Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-13T21:17:57.883800881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:17:58.283831819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:17:58.293169212Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:17:58.293195172Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:17:58.293233175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:17:58.301888498Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:17:59.227038124Z caller=checkpoint.go:372 msg="writing series" size="958 kB"
level=debug ts=2026-05-13T21:17:59.227061556Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-05-13T21:18:02.519171006Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-13T21:18:02.883442324Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:18:02.883471986Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:18:02.883479975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:18:02.883481184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:18:04.309794051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:18:05.811434032Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-13T21:18:07.883407063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:18:08.100757198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44858"
level=debug ts=2026-05-13T21:18:08.254423477Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:18:09.104245814Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-13T21:18:10.367488325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:18:10.377943899Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:18:12.397432042Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=info ts=2026-05-13T21:18:12.739388391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:18:12.747826559Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:18:12.804174976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:18:12.811779412Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:18:12.883578104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:18:15.689985092Z caller=checkpoint.go:372 msg="writing series" size="8.6 kB"
ts=2026-05-13T21:18:15.911282734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46142"
level=warn ts=2026-05-13T21:18:17.883612536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:18:18.9825981Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T21:18:22.275295795Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T21:18:22.883606622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:18:25.568314519Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T21:18:27.883403581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:18:28.860156101Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-13T21:18:32.153736389Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:18:32.153758836Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-13T21:18:32.864642757Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:18:32.866211781Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:18:32.88356552Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:18:32.883584784Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:18:32.883593197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:18:32.88361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:18:34.31819777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:18:35.448127411Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T21:18:35.448152451Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
ts=2026-05-13T21:18:35.738886949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:35460"
level=debug ts=2026-05-13T21:18:36.528364182Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:18:37.883710889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:18:38.104103991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59274"
level=info ts=2026-05-13T21:18:38.592489165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:18:38.603741904Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:18:38.738126799Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-13T21:18:42.031167656Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-13T21:18:42.883223933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:18:45.324306508Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-13T21:18:45.914651375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59314"
level=debug ts=2026-05-13T21:18:46.484422053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:18:47.884201363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:18:48.616934762Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-05-13T21:18:51.908937553Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-13T21:18:52.884153884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:18:55.202448501Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T21:18:56.453317344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:18:56.461799772Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:18:57.88349219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:18:58.495176641Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-13T21:18:59.820069479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:18:59.827415922Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:19:01.787141172Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=debug ts=2026-05-13T21:19:01.787163563Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-13T21:19:02.883998544Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:19:02.884039492Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:19:02.884048236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:19:02.884041548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:19:04.322433345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:19:05.080207275Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-13T21:19:07.511509751Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:19:07.88404861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:19:08.373191434Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=debug ts=2026-05-13T21:19:11.665759633Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-13T21:19:12.883829754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:19:14.95779045Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T21:19:17.883977626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:19:18.251481399Z caller=checkpoint.go:372 msg="writing series" size="445 kB"
level=debug ts=2026-05-13T21:19:21.543560515Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:19:21.543582477Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-13T21:19:22.883323039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:19:24.836102037Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=warn ts=2026-05-13T21:19:27.884058856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:19:28.130711262Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:19:28.130732442Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=debug ts=2026-05-13T21:19:31.420970021Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:19:32.864855157Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:19:32.865349222Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:19:32.883835042Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:19:32.883836091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:19:32.883854029Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:19:32.883861287Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:19:34.32507675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:19:34.714592361Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-13T21:19:36.70548022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:19:36.713104974Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:19:36.821890697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:19:36.823338657Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T21:19:37.739904154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:19:37.883815906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:19:38.007086008Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-13T21:19:41.299509362Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=warn ts=2026-05-13T21:19:42.883958676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:19:44.592239485Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
ts=2026-05-13T21:19:45.918774554Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55730"
level=info ts=2026-05-13T21:19:47.25959039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:19:47.271204272Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:19:47.884152931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:19:47.885228131Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=debug ts=2026-05-13T21:19:51.178068547Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-13T21:19:51.178090073Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-13T21:19:52.883300085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:19:53.405768884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:19:53.413907222Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:19:54.470281632Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T21:19:57.763617303Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=warn ts=2026-05-13T21:19:57.883474567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:20:01.05566661Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=info ts=2026-05-13T21:20:02.883559586Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:20:02.883588534Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:20:02.883596252Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:20:02.883639105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:20:04.328475828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:20:04.348137899Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-13T21:20:05.744544642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38744"
level=debug ts=2026-05-13T21:20:07.424575496Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:20:07.641373958Z caller=checkpoint.go:372 msg="writing series" size="481 kB"
level=debug ts=2026-05-13T21:20:07.64139694Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-13T21:20:07.883566158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:20:08.109906998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35308"
level=debug ts=2026-05-13T21:20:10.933383771Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T21:20:12.883407163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:20:14.226309715Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-13T21:20:17.518822189Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:20:17.88352964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:20:20.812206358Z caller=checkpoint.go:372 msg="writing series" size="479 kB"
level=warn ts=2026-05-13T21:20:22.883387358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:20:24.104605183Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T21:20:24.104643498Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-13T21:20:27.398051391Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=warn ts=2026-05-13T21:20:27.883990486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:20:30.689446632Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T21:20:32.865198398Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:20:32.865652894Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:20:32.884132965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:20:32.88415122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:20:32.884160472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:20:32.884177963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:20:33.983048878Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=info ts=2026-05-13T21:20:34.040078547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:20:34.047867357Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T21:20:34.33126263Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:20:34.45910572Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:20:36.514116849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:20:37.27542075Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=debug ts=2026-05-13T21:20:37.275444188Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-13T21:20:37.883584798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:20:38.112962048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53262"
level=debug ts=2026-05-13T21:20:40.567915862Z caller=checkpoint.go:372 msg="writing series" size="668 kB"
level=warn ts=2026-05-13T21:20:42.883529812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:20:43.22904467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:20:43.236638304Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:20:43.347834227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:20:43.354915019Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:20:43.860926724Z caller=checkpoint.go:372 msg="writing series" size="491 kB"
level=debug ts=2026-05-13T21:20:43.860950416Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-13T21:20:47.153289112Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-13T21:20:47.883539455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:20:50.445689873Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T21:20:52.884071173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:20:53.738883081Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:20:57.037112104Z caller=checkpoint.go:372 msg="writing series" size="9.6 MB"
level=debug ts=2026-05-13T21:20:57.037134048Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.7 MB" series=4
level=warn ts=2026-05-13T21:20:57.883991369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:00.323911923Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T21:21:02.883601349Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:21:02.883600869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:02.883628066Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:21:02.883636182Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:21:03.617133882Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-13T21:21:04.333631294Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:21:06.909364594Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T21:21:07.884250543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:08.043656451Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:21:10.201887078Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-13T21:21:12.88361718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:13.495248037Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-13T21:21:16.78732902Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T21:21:17.883210306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:20.081844049Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T21:21:20.081866473Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=7
level=warn ts=2026-05-13T21:21:22.88407556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:23.372652377Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:21:26.665797574Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-13T21:21:27.884185976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:21:27.936308973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:21:27.944389228Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:21:27.944412256Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:21:27.944441273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:21:27.952112612Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:21:29.925729142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:21:29.958089213Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=debug ts=2026-05-13T21:21:32.864726383Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:21:32.864724285Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T21:21:32.884130342Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:21:32.884157041Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:21:32.884164549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:21:32.884170479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:33.251103826Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=info ts=2026-05-13T21:21:33.624335755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:21:33.632289457Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T21:21:34.337203613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:21:36.543212356Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=warn ts=2026-05-13T21:21:37.883839728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:38.699786993Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:21:39.83652274Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-13T21:21:42.883512186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:21:43.066435805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:21:43.075912656Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:21:43.075945775Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:21:43.075971744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:21:43.082772961Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:21:43.128626882Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-13T21:21:45.92490871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:46670"
level=info ts=2026-05-13T21:21:46.178965011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:21:46.186502595Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:21:46.421089279Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-13T21:21:47.88404562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:49.714552765Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-13T21:21:52.883210736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:53.006603031Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-13T21:21:56.299947049Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-13T21:21:56.616656539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:21:56.626127802Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:21:57.88384304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:21:59.592332038Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-13T21:21:59.592354912Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=12
level=info ts=2026-05-13T21:22:02.883934566Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:22:02.883962732Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:22:02.883970611Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:22:02.883984591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:22:02.891003708Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-13T21:22:02.893445749Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009123686s
ts=2026-05-13T21:22:04.340413666Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:22:05.213155028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:22:07.883725702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:22:08.117884004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54336"
level=warn ts=2026-05-13T21:22:12.883257684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:22:15.928318653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:59702"
level=warn ts=2026-05-13T21:22:17.88406297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:22:19.623818882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:22:19.633421645Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:22:22.883803168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:22:25.86281341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:22:25.870104488Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:22:26.307266981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:22:26.314486783Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:22:26.94380954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:22:27.883773283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:22:32.864637059Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:22:32.865332131Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:22:32.883820513Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T21:22:32.883820342Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-13T21:22:32.88383739Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:22:32.883844713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:22:32.883856713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:22:32.883949559Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-13T21:22:32.883957487Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:22:32.883976481Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.224µs success=true
level=debug ts=2026-05-13T21:22:32.884167047Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T21:22:34.343138305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:22:35.753182477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38304"
level=debug ts=2026-05-13T21:22:36.178446696Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-13T21:22:36.178467389Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=2
level=warn ts=2026-05-13T21:22:37.883935979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:22:38.280147618Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:22:39.469733152Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:22:42.763214988Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=warn ts=2026-05-13T21:22:42.883181264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:22:46.056001778Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T21:22:46.056043578Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=warn ts=2026-05-13T21:22:47.884186081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:22:49.348088245Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=debug ts=2026-05-13T21:22:52.679819347Z caller=checkpoint.go:372 msg="writing series" size="45 MB"
level=debug ts=2026-05-13T21:22:52.679844659Z caller=checkpoint.go:384 msg="flushing series" totalSize="45 MB" series=2
level=warn ts=2026-05-13T21:22:52.883816663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:22:55.933706936Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-13T21:22:57.883586747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:22:59.227106594Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T21:22:59.227135112Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-13T21:23:02.224528195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:23:02.518449012Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-13T21:23:02.883963132Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:23:02.883994025Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:23:02.884001647Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:23:02.884033648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:23:04.34579034Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:23:05.756074393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56850"
level=debug ts=2026-05-13T21:23:05.811522743Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-13T21:23:07.884002466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:23:08.122364709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50918"
level=debug ts=2026-05-13T21:23:09.103965753Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-13T21:23:11.514067232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:23:11.523318211Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:23:12.397047302Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-13T21:23:12.883851757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:23:13.307479922Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:23:13.820132918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:23:13.828689748Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:23:15.69007949Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
level=warn ts=2026-05-13T21:23:17.883798677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:23:18.735373595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:23:18.743788018Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:23:18.982665392Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-13T21:23:22.275037196Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T21:23:22.88350639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:23:25.567994765Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T21:23:27.884004626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:23:28.860142655Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-13T21:23:32.153424531Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:23:32.153447646Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-13T21:23:32.864923354Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:23:32.865407248Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:23:32.883829557Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:23:32.883845091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:23:32.883852299Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:23:32.883871615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:23:34.348858253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:23:35.447339564Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-13T21:23:35.44736437Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
ts=2026-05-13T21:23:35.759234194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60774"
level=debug ts=2026-05-13T21:23:37.628411124Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:23:37.628472205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="412 kB" avg_comp="412 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:23:37.633039295Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:23:37.883470965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:23:38.124970336Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54634"
level=debug ts=2026-05-13T21:23:38.738374218Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T21:23:42.031156735Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-13T21:23:42.883803728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:23:45.323628744Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-13T21:23:45.933794126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35136"
level=warn ts=2026-05-13T21:23:47.884063471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:23:48.616071877Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-05-13T21:23:50.748538354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:23:50.750457484Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T21:23:51.908758669Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-13T21:23:52.884101804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:23:53.930146141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:23:53.93414117Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:23:55.201660448Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=warn ts=2026-05-13T21:23:57.883996703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:23:58.494150872Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-13T21:24:01.786911461Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-13T21:24:01.786935872Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-13T21:24:02.88356777Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:24:02.883596508Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T21:24:02.883590676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:24:02.883606763Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:24:04.35239967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T21:24:04.442008698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:24:04.45372328Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:24:05.079806334Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-13T21:24:07.883605874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:24:08.258517817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:24:08.266840627Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:24:08.37259644Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-13T21:24:09.078058054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:24:09.084854983Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:24:10.268587279Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:24:11.665531003Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-13T21:24:12.883533005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:24:14.958317693Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T21:24:17.883425464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:24:18.251140906Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=debug ts=2026-05-13T21:24:21.543356402Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:24:21.543380212Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-13T21:24:22.883387141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:24:24.836119312Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-13T21:24:27.883978768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:24:28.129637013Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:24:28.129663217Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=debug ts=2026-05-13T21:24:30.026418579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:24:30.747383272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:24:30.751228684Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:24:31.421660516Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:24:32.864612378Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:24:32.865295184Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:24:32.883761599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:24:32.883778451Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:24:32.883786021Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:24:32.883791489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:24:34.355549559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:24:34.714162885Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=warn ts=2026-05-13T21:24:37.883621885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:24:38.006980664Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-13T21:24:38.12970194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49712"
level=debug ts=2026-05-13T21:24:40.09021139Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:24:41.299578169Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=warn ts=2026-05-13T21:24:42.883336026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:24:44.592137187Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-13T21:24:47.884146568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:24:47.884269639Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:24:48.83400255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:24:48.842125979Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T21:24:51.173041306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:24:51.177647169Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=error ts=2026-05-13T21:24:51.180462397Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:24:51.180480554Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:24:51.180509613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:24:51.187425535Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:24:52.883618048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:24:54.4696116Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T21:24:54.469648146Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-13T21:24:57.763387376Z caller=checkpoint.go:372 msg="writing series" size="516 kB"
level=warn ts=2026-05-13T21:24:57.883948886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:25:01.05563527Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-13T21:25:02.259076687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:25:02.269542978Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:25:02.883682356Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:25:02.883686633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:25:02.883709433Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:25:02.883717368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:25:04.347864456Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-13T21:25:04.35805547Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:25:05.764760006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48028"
level=debug ts=2026-05-13T21:25:07.640955228Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:25:07.640980594Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-13T21:25:07.884006835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:25:08.67448493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:25:08.681849219Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:25:10.933982308Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T21:25:12.883847175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:25:13.366933405Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:25:13.550246801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:25:13.55488158Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:25:14.226604197Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-13T21:25:17.518413084Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:25:17.883945887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:25:20.811186811Z caller=checkpoint.go:372 msg="writing series" size="414 kB"
level=debug ts=2026-05-13T21:25:20.963432826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:25:22.884084077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:25:24.104882558Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T21:25:27.397592201Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=debug ts=2026-05-13T21:25:27.397618651Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=6
level=warn ts=2026-05-13T21:25:27.883967238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:25:30.689766933Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T21:25:31.61163774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:25:31.620186199Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T21:25:32.864732917Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:25:32.864735887Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:25:32.883174515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:25:32.883191633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T21:25:32.883188107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:25:32.883199487Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:25:33.982197628Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-13T21:25:34.361392888Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:25:35.767687188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53724"
level=debug ts=2026-05-13T21:25:37.274684571Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-13T21:25:37.883981323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:25:38.133836437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39100"
level=info ts=2026-05-13T21:25:39.811970221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:25:39.819543233Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:25:40.568200624Z caller=checkpoint.go:372 msg="writing series" size="667 kB"
level=debug ts=2026-05-13T21:25:41.506397469Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:25:42.88386459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:25:43.860392628Z caller=checkpoint.go:372 msg="writing series" size="469 kB"
level=debug ts=2026-05-13T21:25:43.860415759Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-13T21:25:47.152972356Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-13T21:25:47.883377341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:25:50.445873149Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T21:25:52.883963425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:25:53.737870038Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:25:57.041738331Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-13T21:25:57.041760167Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=4
level=info ts=2026-05-13T21:25:57.680994618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:25:57.688324714Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:25:57.884187295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:00.323878897Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T21:26:02.883440875Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:26:02.88346808Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:26:02.883477527Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:26:02.883487335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:03.616534166Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-13T21:26:03.964132554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:26:03.970918716Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T21:26:04.364394893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:26:04.925532296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:26:06.909166782Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-13T21:26:07.143874449Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:26:07.884206559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:10.201970468Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-13T21:26:12.883214438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:13.494750769Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
ts=2026-05-13T21:26:15.942631107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:38258"
level=debug ts=2026-05-13T21:26:16.787351766Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T21:26:17.883823274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:20.081995245Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-13T21:26:20.082033772Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=info ts=2026-05-13T21:26:20.414148808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:26:20.423903707Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:26:22.883268694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:23.372857939Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T21:26:25.816835517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:26:25.824860795Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:26:26.665557321Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T21:26:27.883518014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:29.957676093Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T21:26:32.864659487Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:26:32.865637062Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-13T21:26:32.884169803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:26:32.884192695Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:26:32.884211911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:26:32.88421982Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:26:33.250899384Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
ts=2026-05-13T21:26:34.367235854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:26:36.291101901Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:26:36.542873252Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-13T21:26:37.884008876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:39.836169217Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-13T21:26:42.883219897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:43.128769524Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-05-13T21:26:46.420927486Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-13T21:26:47.883830657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:49.010994947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:26:49.713788079Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-13T21:26:52.51698959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:26:52.525819733Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:26:52.883427071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:53.007080026Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-13T21:26:55.13866894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:26:55.143409484Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:26:56.299891849Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-05-13T21:26:57.883646786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:26:59.592298688Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-13T21:26:59.592321469Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=12
level=info ts=2026-05-13T21:27:02.883585442Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:27:02.883633594Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:27:02.883644253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:27:02.88367706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:27:02.891410371Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-13T21:27:02.893556723Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009438665s
ts=2026-05-13T21:27:04.370600385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=warn ts=2026-05-13T21:27:07.884076707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:27:11.806125499Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:27:12.883789331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:27:13.047151196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:27:13.055731054Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:27:17.384922047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:27:17.395082547Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:27:17.883902358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:27:22.883906385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:27:27.883213578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:27:28.335435875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:27:28.33707333Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T21:27:32.864681548Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:27:32.865236381Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:27:32.883650606Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T21:27:32.883656279Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-13T21:27:32.883667919Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:27:32.883675197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:27:32.88370085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:27:32.883805937Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-13T21:27:32.883813279Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:27:32.883829981Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.911µs success=true
level=debug ts=2026-05-13T21:27:32.883999203Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T21:27:34.373743052Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:27:35.773785181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59798"
level=debug ts=2026-05-13T21:27:36.178919633Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-13T21:27:36.178942283Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=2
level=warn ts=2026-05-13T21:27:37.883778049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:27:39.470403045Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:27:41.734057578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:27:42.762987366Z caller=checkpoint.go:372 msg="writing series" size="433 kB"
level=warn ts=2026-05-13T21:27:42.883819407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:27:45.371298292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.0 MB" avg_comp="2.0 MB" total_uncomp="69 MB" avg_uncomp="17 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:27:45.380701702Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:27:46.056341795Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T21:27:46.056363403Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=warn ts=2026-05-13T21:27:47.88339367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:27:49.348466275Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-13T21:27:52.036387359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:27:52.042581334Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:27:52.678958141Z caller=checkpoint.go:372 msg="writing series" size="45 MB"
level=debug ts=2026-05-13T21:27:52.678985141Z caller=checkpoint.go:384 msg="flushing series" totalSize="45 MB" series=2
level=warn ts=2026-05-13T21:27:52.883516644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:27:55.933157828Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-13T21:27:57.883775644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:27:59.227461867Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T21:27:59.227485062Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-13T21:28:02.519225474Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-05-13T21:28:02.883846525Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:28:02.883850482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:28:02.883877914Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:28:02.883887729Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:28:04.376636704Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:28:05.811271299Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-13T21:28:07.0941757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:28:07.104997967Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:28:07.105045688Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:28:07.105070423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:28:07.112752Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:28:07.883383534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:28:09.104033091Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-13T21:28:09.80823541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:28:09.808281899Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T21:28:09.814690446Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:28:10.394497091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:28:10.402320599Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:28:12.396720654Z caller=checkpoint.go:372 msg="writing series" size="639 B"
level=warn ts=2026-05-13T21:28:12.883810942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:28:15.689110497Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-13T21:28:17.8836335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:28:18.982127666Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-13T21:28:22.274688533Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T21:28:22.88400293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:28:25.567095642Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T21:28:26.19598374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:28:26.204645186Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:28:26.950819402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:28:26.958163202Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:28:27.883709226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:28:28.860259465Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-13T21:28:30.239623311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:28:31.886265387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:28:31.893347991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:28:31.893370244Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:28:31.893396784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:28:31.902075288Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:28:32.153341435Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:28:32.153368108Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-13T21:28:32.86465741Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:28:32.86496776Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:28:32.883353669Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:28:32.883374356Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:28:32.883384087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:28:32.883384084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:28:34.380204177Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:28:35.44650078Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T21:28:35.446522198Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=1
level=info ts=2026-05-13T21:28:35.465055922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:28:35.469169855Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:28:37.032681947Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:28:37.883494397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:28:38.143391791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50620"
level=debug ts=2026-05-13T21:28:38.738506789Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T21:28:42.03127614Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-13T21:28:42.883356595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:28:44.937702063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:28:44.9460002Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:28:45.324090896Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T21:28:47.883565102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:28:48.616086014Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-05-13T21:28:51.909300346Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-13T21:28:52.883541809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:28:54.702653065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:28:54.710639384Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:28:55.20241469Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T21:28:57.201068295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:28:57.209842629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T21:28:57.883198912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:28:58.494074657Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-13T21:29:01.786800271Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-13T21:29:01.786823841Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-13T21:29:02.883713258Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:29:02.883712048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:02.883743654Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:29:02.883753425Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:29:04.383482868Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:29:05.080123339Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T21:29:06.415257804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:29:06.42357135Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:29:07.883364085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:08.372453888Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-13T21:29:09.381674353Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:29:11.664832773Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-13T21:29:12.88389824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:14.957890443Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T21:29:17.884080979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:18.251095811Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=info ts=2026-05-13T21:29:20.307339309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:29:20.317967416Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:29:21.542994398Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:29:21.543035672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-13T21:29:22.883232222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:24.835821677Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-13T21:29:27.88420456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:28.132091763Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:29:28.13211381Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=debug ts=2026-05-13T21:29:28.813687824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:29:31.420991102Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T21:29:32.864410446Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:29:32.865456866Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-13T21:29:32.883861996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:29:32.883876809Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:29:32.883896061Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:29:32.883914706Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:29:33.560162847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:29:33.566315034Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T21:29:34.386222331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:29:34.713756672Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
ts=2026-05-13T21:29:35.780889777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52472"
level=info ts=2026-05-13T21:29:37.830777923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:29:37.836977643Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T21:29:37.883692823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:38.006003766Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-05-13T21:29:41.298897087Z caller=checkpoint.go:372 msg="writing series" size="470 kB"
level=info ts=2026-05-13T21:29:42.250148484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:29:42.258276989Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:29:42.883414741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:44.591800874Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-13T21:29:45.325307013Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:29:47.774073016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:29:47.782990221Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:29:47.883784141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:47.884846274Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-13T21:29:51.177399491Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-13T21:29:51.177422069Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-13T21:29:52.883712422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:29:54.469650085Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T21:29:57.763132904Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
level=warn ts=2026-05-13T21:29:57.883866057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:30:01.055633307Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=info ts=2026-05-13T21:30:02.883174938Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:30:02.883204202Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:30:02.883213541Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:30:02.883217795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:30:04.348113554Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-13T21:30:04.389361625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:30:07.641279123Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=debug ts=2026-05-13T21:30:07.641303308Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-05-13T21:30:07.883332018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:30:10.933454599Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-13T21:30:12.883545204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:30:14.225733966Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T21:30:15.46381827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:30:15.476266013Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:30:16.589407257Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:30:17.518214573Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:30:17.883686816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:30:18.551535427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:30:18.555999923Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:30:18.556038779Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:30:18.55606835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:30:18.559858316Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:30:20.811915888Z caller=checkpoint.go:372 msg="writing series" size="414 kB"
level=warn ts=2026-05-13T21:30:22.883518634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:30:23.543693121Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:30:24.104728954Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T21:30:27.39744153Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=debug ts=2026-05-13T21:30:27.39746324Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=6
level=warn ts=2026-05-13T21:30:27.883198727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:30:27.942424806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:30:27.947948308Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:30:30.689352949Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T21:30:32.864863968Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:30:32.865280061Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=debug ts=2026-05-13T21:30:32.866522884Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T21:30:32.866569278Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976341 n=1
level=debug ts=2026-05-13T21:30:32.866612406Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976341 n=1
level=info ts=2026-05-13T21:30:32.883559854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:30:32.8835752Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:30:32.883596458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:30:32.88361509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:30:33.9826145Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T21:30:34.11495427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:30:34.128401715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:30:34.310149544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:30:34.317184035Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T21:30:34.392604525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:30:37.274483909Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-13T21:30:37.883682875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:30:38.10085342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:30:38.105117802Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:30:40.568108887Z caller=checkpoint.go:372 msg="writing series" size="703 kB"
level=warn ts=2026-05-13T21:30:42.884030336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:30:43.86019096Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:30:43.860213937Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
ts=2026-05-13T21:30:45.956987023Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60714"
level=debug ts=2026-05-13T21:30:46.531130914Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:30:47.153109649Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-13T21:30:47.883519955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:30:50.445688486Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T21:30:52.883397294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:30:53.737690145Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T21:30:56.337429634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:30:56.3458012Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:30:57.041092673Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-13T21:30:57.041115453Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=warn ts=2026-05-13T21:30:57.883830063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:00.324120201Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-13T21:31:01.202902265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:31:02.88343264Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:31:02.883465524Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:31:02.883473696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:31:02.883475536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:03.616455304Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
ts=2026-05-13T21:31:04.395339585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:31:06.909457287Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T21:31:07.883778647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:10.201100498Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-13T21:31:12.883374855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:13.495884629Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T21:31:13.495910865Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-13T21:31:16.786910086Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-13T21:31:16.997607424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:31:17.003815624Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:31:17.142285577Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:31:17.88404615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:20.080466069Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T21:31:20.080489529Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=info ts=2026-05-13T21:31:22.174563016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:31:22.189908765Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:31:22.189937263Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:31:22.189971851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:31:22.19955253Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:31:22.8831764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:23.372689602Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T21:31:24.050910016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:31:24.057620142Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:31:26.664800353Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T21:31:27.884170478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:31:28.272353968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:31:28.280003899Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:31:29.958004955Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-13T21:31:32.864631314Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:31:32.864794749Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:31:32.884141191Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:31:32.884157606Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:31:32.884165508Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:31:32.884185033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:33.25021144Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
ts=2026-05-13T21:31:34.398513378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:31:36.54289956Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-13T21:31:37.493660171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:31:37.501424341Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:31:37.883806283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:39.83593899Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-13T21:31:42.88341168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:43.128848678Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=info ts=2026-05-13T21:31:46.075260651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:31:46.082813425Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:31:46.421652331Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-13T21:31:47.883365862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:49.713393874Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-13T21:31:52.800860747Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:31:52.883473391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:53.006818577Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-13T21:31:56.299202715Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-13T21:31:56.834322964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:31:56.835899127Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T21:31:57.166062201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:31:57.171839677Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:31:57.88425673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:31:59.59378981Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-13T21:31:59.593818282Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=12
level=info ts=2026-05-13T21:32:02.883140763Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:32:02.88317106Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:32:02.883179734Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:32:02.8832098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:32:02.891431455Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-13T21:32:02.89460018Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010649108s
ts=2026-05-13T21:32:04.400928682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=warn ts=2026-05-13T21:32:07.883689782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:32:10.016314182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:32:10.026006514Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:32:12.883545866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:32:15.31568486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:32:15.321915874Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T21:32:17.883491597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:32:22.883830166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:32:25.912278724Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:32:26.443670387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:32:26.454911752Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:32:27.883413496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:32:32.864681619Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:32:32.865055634Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:32:32.883306408Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T21:32:32.883343552Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:32:32.883363694Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:32:32.88337251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:32:32.88338265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:32:32.883440455Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-13T21:32:32.883447865Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:32:32.883463649Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.25µs success=true
level=debug ts=2026-05-13T21:32:32.883643348Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T21:32:34.404372853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T21:32:34.761839529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:32:34.769231151Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T21:32:35.790497044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39002"
level=debug ts=2026-05-13T21:32:36.179394954Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-13T21:32:36.179418585Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=2
level=warn ts=2026-05-13T21:32:37.883495665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:32:38.155856617Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34590"
level=debug ts=2026-05-13T21:32:39.470100894Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:32:42.679477608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:32:42.761988461Z caller=checkpoint.go:372 msg="writing series" size="507 kB"
level=warn ts=2026-05-13T21:32:42.883582493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:32:46.05568676Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T21:32:46.05570864Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=warn ts=2026-05-13T21:32:47.883780644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:32:49.348008902Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-05-13T21:32:52.681848725Z caller=checkpoint.go:372 msg="writing series" size="46 MB"
level=debug ts=2026-05-13T21:32:52.68189059Z caller=checkpoint.go:384 msg="flushing series" totalSize="46 MB" series=2
level=warn ts=2026-05-13T21:32:52.884040698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:32:53.618757683Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:32:54.268845966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:32:54.276077775Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:32:55.933071328Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-13T21:32:57.88331168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:32:59.226572342Z caller=checkpoint.go:372 msg="writing series" size="978 kB"
level=debug ts=2026-05-13T21:32:59.226596611Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-05-13T21:33:02.518405935Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-05-13T21:33:02.88318139Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:33:02.883209865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:33:02.883218632Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:33:02.883206183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:33:03.921381592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:33:03.931471033Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T21:33:04.407322685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:33:05.793510606Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58350"
level=debug ts=2026-05-13T21:33:05.811129728Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-13T21:33:07.883404384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:33:09.103366162Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-13T21:33:09.113458322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:33:09.123326484Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:33:12.396739703Z caller=checkpoint.go:372 msg="writing series" size="639 B"
level=warn ts=2026-05-13T21:33:12.884087071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:33:15.688747377Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T21:33:15.753870627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:33:15.762676434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:33:17.88354388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:33:18.98193669Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-13T21:33:22.274215106Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T21:33:22.883657111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:33:25.56725383Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-13T21:33:26.309732916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:33:26.321901054Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:33:27.883548109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:33:28.860404346Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-13T21:33:29.453174527Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:33:32.15270605Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:33:32.152727666Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-13T21:33:32.864647291Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:33:32.864650387Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:33:32.884032121Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:33:32.884050828Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:33:32.884061494Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:33:32.884063301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:33:34.410121748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:33:35.446418101Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T21:33:35.446437561Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=1
ts=2026-05-13T21:33:35.796381619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42332"
level=debug ts=2026-05-13T21:33:36.941417796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:33:37.883202481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:33:38.161116632Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33346"
level=debug ts=2026-05-13T21:33:38.738001647Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T21:33:42.030808797Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=warn ts=2026-05-13T21:33:42.883905036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:33:45.323249479Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T21:33:47.883755581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:33:48.025127202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:33:48.030979354Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:33:48.616156391Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-05-13T21:33:51.908834234Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-13T21:33:52.883283222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:33:55.201677366Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T21:33:55.505996896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:33:55.515536078Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:33:56.479954407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:33:56.49077424Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:33:57.883961426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:33:58.494206684Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-13T21:34:00.317481599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:34:00.317519194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="929 kB" avg_comp="929 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:34:00.322690705Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:34:00.635607155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:34:00.641635577Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:34:01.786421874Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-13T21:34:01.786445151Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-13T21:34:02.884153718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:34:02.884151221Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:34:02.884179874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:34:02.884187743Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:34:04.413178055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:34:05.079194086Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-13T21:34:05.799672585Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48588"
level=warn ts=2026-05-13T21:34:07.883557029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:34:08.16381811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43858"
level=debug ts=2026-05-13T21:34:08.37265338Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=debug ts=2026-05-13T21:34:11.665101787Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=info ts=2026-05-13T21:34:12.675891908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:34:12.685321438Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T21:34:12.883511456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:34:14.956865838Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-13T21:34:17.883961764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:34:18.250075779Z caller=checkpoint.go:372 msg="writing series" size="399 kB"
level=debug ts=2026-05-13T21:34:21.542302716Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:34:21.542320938Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-13T21:34:22.883587534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:34:24.835434961Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=info ts=2026-05-13T21:34:24.836408976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:34:24.845217948Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:34:25.695980856Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:34:27.883589278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:34:28.129264293Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:34:28.129284964Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=debug ts=2026-05-13T21:34:30.734703472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:34:31.420796667Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:34:32.827227274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:34:32.83153414Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:34:32.864673656Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:34:32.864679287Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:34:32.88405634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:34:32.884075199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:34:32.884084683Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:34:32.884093565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:34:34.416431669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:34:34.713892057Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
ts=2026-05-13T21:34:35.802955466Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37744"
level=warn ts=2026-05-13T21:34:37.88348199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:34:38.005848389Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-13T21:34:38.445080889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:34:38.452996552Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:34:41.299206718Z caller=checkpoint.go:372 msg="writing series" size="391 kB"
level=warn ts=2026-05-13T21:34:42.884029589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:34:44.302276398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:34:44.311221771Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:34:44.311245398Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:34:44.311272091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:34:44.317722908Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:34:44.591649799Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-05-13T21:34:47.501983483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:34:47.507266787Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T21:34:47.883692942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:34:47.883711797Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-13T21:34:51.177474152Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-13T21:34:51.177497805Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-13T21:34:52.883791606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:34:54.469095177Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T21:34:55.019595341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:34:57.093309553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:34:57.101526221Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:34:57.101552543Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:34:57.101579278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:34:57.108098307Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:34:57.762200687Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
level=warn ts=2026-05-13T21:34:57.883426446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:35:01.055320706Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=info ts=2026-05-13T21:35:02.737040932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:35:02.744535984Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:35:02.883440113Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:35:02.883477204Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:35:02.883485386Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:35:02.883502616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:35:04.347378921Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-13T21:35:04.419657475Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:35:07.640473289Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=debug ts=2026-05-13T21:35:07.640496098Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-13T21:35:07.883766119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:35:08.168366373Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51644"
level=info ts=2026-05-13T21:35:10.774613817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:35:10.782199869Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:35:10.782222023Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:35:10.782245563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:35:10.789398656Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:35:10.933300023Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-13T21:35:11.686129024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:35:11.692935053Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T21:35:12.88398514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:35:14.226266837Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
ts=2026-05-13T21:35:15.972323739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48236"
level=debug ts=2026-05-13T21:35:17.518209766Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:35:17.883775758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:35:19.983471888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:35:19.987860769Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:35:20.810991889Z caller=checkpoint.go:372 msg="writing series" size="414 kB"
level=info ts=2026-05-13T21:35:21.97445471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:35:21.98176271Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:35:22.883997703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:35:24.104150036Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T21:35:24.104170867Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-13T21:35:24.659246333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:35:24.666789296Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:35:24.769161037Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:35:26.272992622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:35:27.397655551Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=warn ts=2026-05-13T21:35:27.883627505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:35:28.489246213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:35:28.497417799Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:35:30.688951882Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T21:35:32.864623733Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:35:32.865002033Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:35:32.883265791Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:35:32.883287804Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T21:35:32.883287734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:35:32.883297824Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:35:33.981687379Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-13T21:35:34.422975606Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T21:35:36.054887143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:35:36.059938419Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:35:37.274606254Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-13T21:35:37.27462826Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-13T21:35:37.883771805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:35:40.567122978Z caller=checkpoint.go:372 msg="writing series" size="760 kB"
level=warn ts=2026-05-13T21:35:42.883175367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:35:43.005467149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:35:43.012456808Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:35:43.859847593Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=debug ts=2026-05-13T21:35:43.85987109Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-13T21:35:47.15274369Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-13T21:35:47.883263615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:35:50.44568903Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-13T21:35:50.540239308Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:35:52.883389706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:35:53.73784288Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:35:57.04322042Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-13T21:35:57.043245684Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=4
level=warn ts=2026-05-13T21:35:57.883545128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:00.323059691Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T21:36:00.879837981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:36:00.887837285Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T21:36:02.883858717Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:36:02.88386439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:02.883885016Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:36:02.883892685Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:36:03.615488354Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-13T21:36:04.107551195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:36:04.111977973Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T21:36:04.425785156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T21:36:04.897887831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:36:04.905333472Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:36:06.908449775Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T21:36:07.883631302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:10.201537556Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-13T21:36:12.883718662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:13.495083212Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T21:36:13.49512433Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
ts=2026-05-13T21:36:15.977153591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33030"
level=debug ts=2026-05-13T21:36:16.786910771Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-13T21:36:17.672182727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:36:17.673925978Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T21:36:17.883989753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:20.080737123Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T21:36:20.08076508Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=debug ts=2026-05-13T21:36:20.471992582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:36:20.554089986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:36:20.561224569Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:36:22.884067551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:23.371845535Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:36:26.664442478Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-13T21:36:27.553852843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:36:27.560841217Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T21:36:27.883229901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:29.957269203Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=info ts=2026-05-13T21:36:30.301155063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:36:30.306916322Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:36:32.864764714Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:36:32.865168797Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:36:32.88345672Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:36:32.883475186Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:36:32.883483065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:36:32.883509062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:33.250033653Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
ts=2026-05-13T21:36:34.428683603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:36:36.543119128Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-13T21:36:37.884044769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:39.835565746Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-13T21:36:42.88380395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:43.128409959Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=debug ts=2026-05-13T21:36:45.680149554Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:36:45.977042468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:36:45.986516432Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:36:46.420711056Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-13T21:36:47.04952226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:36:47.054236624Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:36:47.054254496Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:36:47.054276833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:36:47.057874251Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:36:47.883730487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:36:48.360740062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:36:48.369594968Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:36:49.713744221Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-13T21:36:52.883429236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:53.005897947Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-13T21:36:56.298491144Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-13T21:36:56.29851603Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-13T21:36:57.884076221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:36:59.591752454Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=info ts=2026-05-13T21:37:00.303600986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:37:00.311227748Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:37:01.950614515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:37:01.959428088Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:37:02.883363188Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:37:02.883397078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T21:37:02.883389021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:37:02.883407229Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:37:02.884441924Z caller=checkpoint.go:384 msg="flushing series" totalSize="657 kB" series=1
level=info ts=2026-05-13T21:37:02.889551143Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-13T21:37:02.890638153Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007044528s
ts=2026-05-13T21:37:04.431905957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:37:05.289051746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:37:07.883715951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:37:11.47605968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:37:11.482121807Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T21:37:12.883332599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:37:13.51321774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:37:13.518770809Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:37:15.456706465Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T21:37:15.980762831Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52892"
level=warn ts=2026-05-13T21:37:17.883953079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:37:22.884047464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:37:27.883994452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:37:31.617955472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:37:31.628101684Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T21:37:32.864634585Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:37:32.864686162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:37:32.884031119Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T21:37:32.884068386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:37:32.884088349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T21:37:32.884085583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:37:32.884099163Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:37:32.884154284Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-13T21:37:32.88416151Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:37:32.884178372Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.694µs success=true
level=debug ts=2026-05-13T21:37:32.884358985Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T21:37:34.43461371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:37:36.180066879Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-13T21:37:36.180089508Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=2
level=info ts=2026-05-13T21:37:36.73217127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:37:36.740160796Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:37:36.740184693Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:37:36.740221479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:37:36.748108832Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:37:37.88363834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:37:39.470690604Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-05-13T21:37:39.573870962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:37:39.57837229Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:37:40.517086305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:37:42.763608277Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=warn ts=2026-05-13T21:37:42.883469961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:37:46.056476767Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T21:37:46.05650191Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=warn ts=2026-05-13T21:37:47.883619354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:37:49.348396586Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-13T21:37:52.686401887Z caller=checkpoint.go:372 msg="writing series" size="47 MB"
level=debug ts=2026-05-13T21:37:52.686436859Z caller=checkpoint.go:384 msg="flushing series" totalSize="47 MB" series=2
level=warn ts=2026-05-13T21:37:52.883253728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:37:54.367077817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:37:54.37671332Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:37:54.815403232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:37:54.823193542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:37:55.933976286Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-13T21:37:56.898837069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:37:57.884067226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:37:59.227599082Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-13T21:37:59.227621185Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-13T21:38:02.519011466Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=warn ts=2026-05-13T21:38:02.88369698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:38:02.883694856Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:38:02.883728174Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:38:02.883738604Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:38:04.438200878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T21:38:04.576171683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:38:04.58337158Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:38:05.43267368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:38:05.437833826Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:38:05.811978893Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-13T21:38:07.884055935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:38:08.17830026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41224"
level=debug ts=2026-05-13T21:38:09.10448833Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-13T21:38:12.397107056Z caller=checkpoint.go:372 msg="writing series" size="640 B"
level=warn ts=2026-05-13T21:38:12.884222366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:38:13.27467172Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:38:15.689734818Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T21:38:15.985490929Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:52306"
level=warn ts=2026-05-13T21:38:17.884196261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:38:18.982219375Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-13T21:38:21.664570348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:38:21.674333826Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:38:22.275028555Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T21:38:22.883391271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:38:25.360834328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:38:25.368585246Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:38:25.568144559Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T21:38:27.884119997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:38:28.860432466Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-13T21:38:32.153178496Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:38:32.15320132Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-05-13T21:38:32.864687253Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:38:32.86496419Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:38:32.883239704Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:38:32.883260187Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:38:32.883269895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:38:32.883274683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:38:34.431945048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:38:34.441002755Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:38:35.447172758Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T21:38:35.447197098Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=1
ts=2026-05-13T21:38:35.815154916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:44816"
level=info ts=2026-05-13T21:38:36.554553558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:38:36.565970884Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:38:37.639651696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:38:37.64590392Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:38:37.884247198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:38:38.18121054Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58544"
level=debug ts=2026-05-13T21:38:38.73840679Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T21:38:39.679628624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:38:42.03184852Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-13T21:38:42.883863798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:38:45.323882702Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T21:38:47.883334623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:38:48.616852721Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-05-13T21:38:51.908903395Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-13T21:38:52.884072126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:38:54.616219749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:38:54.622558458Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:38:55.202569036Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=warn ts=2026-05-13T21:38:57.883561805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:38:58.495297942Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-13T21:39:00.499758837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:39:00.505868567Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:39:01.787578587Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-13T21:39:01.787604183Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-13T21:39:02.883324256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:39:02.883355598Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:39:02.883364013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:39:02.883377582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:39:04.443608233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:39:05.08000912Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-13T21:39:06.824428826Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:39:07.883932836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:39:08.37284544Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-13T21:39:11.665006691Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-13T21:39:12.883915649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:39:14.95783072Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T21:39:15.090259706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:39:15.107938105Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-13T21:39:15.989661455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40358"
level=warn ts=2026-05-13T21:39:17.88364408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:39:18.250862976Z caller=checkpoint.go:372 msg="writing series" size="541 kB"
level=debug ts=2026-05-13T21:39:18.250889984Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-13T21:39:20.118294756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:39:20.126110222Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:39:21.141776674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:39:21.145967313Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:39:21.543141306Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-05-13T21:39:21.838822235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:39:21.846247571Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:39:22.884089249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:39:24.83658854Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=warn ts=2026-05-13T21:39:27.883767164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:39:28.132700903Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:39:28.132721614Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=3
level=debug ts=2026-05-13T21:39:31.421822967Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-13T21:39:32.072104502Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:39:32.864239146Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:39:32.865291418Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T21:39:32.883589133Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:39:32.883606971Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:39:32.883614853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:39:32.883665246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:39:34.446346667Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:39:34.714838509Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=warn ts=2026-05-13T21:39:37.884028407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:39:38.007360163Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
ts=2026-05-13T21:39:38.185514218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57346"
level=debug ts=2026-05-13T21:39:38.574240612Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:39:41.299266703Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=warn ts=2026-05-13T21:39:42.8840265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:39:44.592067602Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
ts=2026-05-13T21:39:45.993166846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:53040"
level=info ts=2026-05-13T21:39:47.547465027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:39:47.55517477Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T21:39:47.883560498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:39:47.884685117Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-13T21:39:51.177358243Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=debug ts=2026-05-13T21:39:51.177382319Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-13T21:39:52.883182536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:39:54.470687605Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-13T21:39:56.719920667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:39:56.727412525Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:39:57.763604344Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=warn ts=2026-05-13T21:39:57.883803838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:40:00.486312853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:40:00.494898609Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:40:01.055526129Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-13T21:40:02.00407311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:40:02.008380465Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:40:02.883276546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:40:02.8832823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:40:02.883305349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:40:02.883313916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:40:04.348520351Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-13T21:40:04.348542971Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-13T21:40:04.449868788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T21:40:04.492756663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:40:04.500992645Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:40:07.641132803Z caller=checkpoint.go:372 msg="writing series" size="632 kB"
level=debug ts=2026-05-13T21:40:07.816431285Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:40:07.884258777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:40:10.650596579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:40:10.658723481Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:40:10.93359392Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-13T21:40:12.883912406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:40:14.226564122Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=debug ts=2026-05-13T21:40:17.518868828Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:40:17.883512181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:40:20.812069887Z caller=checkpoint.go:372 msg="writing series" size="474 kB"
level=debug ts=2026-05-13T21:40:20.812093553Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=warn ts=2026-05-13T21:40:22.883335928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:40:23.776563149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:40:23.778143531Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T21:40:24.10539118Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T21:40:27.397677083Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=debug ts=2026-05-13T21:40:27.397697903Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-13T21:40:27.883982262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:40:30.69021966Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T21:40:32.864710029Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:40:32.86531519Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:40:32.883677224Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:40:32.883695573Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:40:32.883703715Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:40:32.883718748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:40:33.982959581Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-13T21:40:34.45248827Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:40:34.699832348Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:40:37.274922918Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=warn ts=2026-05-13T21:40:37.883816644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:40:40.568402766Z caller=checkpoint.go:372 msg="writing series" size="618 kB"
level=warn ts=2026-05-13T21:40:42.883869497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:40:43.860878247Z caller=checkpoint.go:372 msg="writing series" size="383 kB"
level=debug ts=2026-05-13T21:40:43.860899823Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-13T21:40:44.221523675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:40:44.228361938Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T21:40:44.228380357Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:40:44.228403493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:40:44.233347063Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:40:45.435670474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:40:45.443342489Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:40:47.153690064Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=info ts=2026-05-13T21:40:47.6519891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:40:47.656414804Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:40:47.883475846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:40:50.445532157Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-13T21:40:50.901267147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:40:50.909425671Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T21:40:52.883485394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:40:53.738476515Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:40:57.040668032Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-13T21:40:57.040691522Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=4
level=warn ts=2026-05-13T21:40:57.883569398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:40:58.214674658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:40:58.223381983Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:40:58.223405893Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:40:58.22343297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:40:58.229690412Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:41:00.323834804Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-13T21:41:02.883251233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:41:02.883276497Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:41:02.883309081Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:41:02.883318507Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:41:03.616762596Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-13T21:41:03.671065892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:41:03.676062819Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:41:03.899542583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:41:03.906803268Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T21:41:04.456135002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:41:06.350333348Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:41:06.908910896Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T21:41:07.883821943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:41:08.192221136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49682"
level=info ts=2026-05-13T21:41:10.11997992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:41:10.127208268Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:41:10.202432306Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-13T21:41:12.883205893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:41:13.495948356Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T21:41:13.495969529Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-13T21:41:16.787504216Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T21:41:17.883849072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:41:20.080738829Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-13T21:41:20.080760397Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=warn ts=2026-05-13T21:41:22.883984235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:41:23.372889983Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:41:24.916692566Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:41:26.665732751Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T21:41:27.883441263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:41:29.957823439Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-13T21:41:30.253475718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:41:30.257896538Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:41:30.830424794Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:41:31.254806438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:41:31.259348189Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:41:32.864621265Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:41:32.864862544Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:41:32.883141926Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:41:32.883165965Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:41:32.883176434Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:41:32.883184333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:41:33.210870572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:41:33.21876829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:41:33.250965208Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
ts=2026-05-13T21:41:34.459235474Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:41:36.543375754Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=warn ts=2026-05-13T21:41:37.884219686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:41:39.084666131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:41:39.092041136Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:41:39.092064461Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:41:39.092088931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:41:39.099135676Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:41:39.836425113Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-13T21:41:42.883617419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:41:43.128869992Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=debug ts=2026-05-13T21:41:46.421213352Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-13T21:41:47.883966652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:41:49.271805393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:41:49.279641292Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:41:49.279662371Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:41:49.279686037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:41:49.285423279Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:41:49.714494925Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-13T21:41:50.038468549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:41:50.04671581Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:41:52.883538471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:41:53.00693336Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-13T21:41:53.651536844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:41:53.660780989Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:41:56.29980263Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-13T21:41:57.883805431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:41:59.592958446Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-13T21:41:59.592984826Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=12
level=info ts=2026-05-13T21:42:00.715773373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:42:00.725182396Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:42:02.883353319Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:42:02.883369273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:42:02.88338193Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:42:02.883389808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:42:02.8927889Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-13T21:42:02.893915857Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009607435s
ts=2026-05-13T21:42:04.462327178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T21:42:05.007068712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:42:05.007121633Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T21:42:05.01327304Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T21:42:07.883366705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:42:12.883595093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:42:12.93739463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:42:15.172407163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:42:15.177461004Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T21:42:17.883924774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:42:18.291076602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:42:18.295471541Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:42:19.266651633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:42:19.272173788Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T21:42:20.939312269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:42:20.946340779Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:42:21.461969965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:42:21.469291452Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:42:22.883608955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:42:27.884216687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:42:29.357612839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:42:32.865333187Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:42:32.865770015Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:42:32.88403662Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T21:42:32.884046403Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:42:32.884063364Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:42:32.884071386Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:42:32.884083575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:42:32.884146489Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-13T21:42:32.884154533Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:42:32.88417017Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.544µs success=true
level=debug ts=2026-05-13T21:42:32.884354604Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-13T21:42:34.465042912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T21:42:35.880949631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:42:35.890340369Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:42:36.17974501Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:42:36.179768088Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=2
level=warn ts=2026-05-13T21:42:37.88331956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:42:38.197250969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59022"
level=debug ts=2026-05-13T21:42:39.470239916Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-05-13T21:42:40.110826999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:42:40.119360093Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:42:42.762901364Z caller=checkpoint.go:372 msg="writing series" size="505 kB"
level=warn ts=2026-05-13T21:42:42.883859638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:42:46.003221637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:60130"
level=debug ts=2026-05-13T21:42:46.057288175Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T21:42:46.057311594Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=3
level=info ts=2026-05-13T21:42:46.985267397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:42:46.992212948Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:42:47.211418003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:42:47.883985807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:42:49.348680834Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=debug ts=2026-05-13T21:42:52.683245729Z caller=checkpoint.go:372 msg="writing series" size="49 MB"
level=debug ts=2026-05-13T21:42:52.683273651Z caller=checkpoint.go:384 msg="flushing series" totalSize="49 MB" series=2
level=warn ts=2026-05-13T21:42:52.883311888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:42:54.004008741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:42:55.933416218Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-13T21:42:57.883708636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:42:59.165512203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:42:59.173216378Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:42:59.227285136Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T21:42:59.227306652Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-13T21:43:02.518888759Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-05-13T21:43:02.88373469Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:43:02.883764201Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:43:02.883772582Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:43:02.883781409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:43:04.46826866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:43:05.811442665Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
ts=2026-05-13T21:43:05.828731848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37478"
level=info ts=2026-05-13T21:43:06.986959488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:43:06.997229311Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:43:07.884047151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:43:09.103952521Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-13T21:43:11.075135368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:43:11.079784803Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:43:11.079803823Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:43:11.079826643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:43:11.083429484Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:43:12.397353343Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T21:43:12.883368698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:43:14.457032007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:43:14.46461357Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:43:15.69002812Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-13T21:43:17.883237351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:43:18.982232693Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-13T21:43:19.664231982Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:43:22.275495408Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-13T21:43:22.884065236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:43:24.67170603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:43:25.568221478Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-13T21:43:27.88320812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:43:28.089220016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:43:28.094981637Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:43:28.860221607Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=debug ts=2026-05-13T21:43:32.153598601Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:43:32.153623192Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-05-13T21:43:32.864746014Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:43:32.864765821Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:43:32.884056417Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:43:32.884069973Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:43:32.884077568Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:43:32.884113235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:43:34.471850879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T21:43:35.172305836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:43:35.179985176Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:43:35.447361056Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-13T21:43:35.447382735Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=1
level=info ts=2026-05-13T21:43:36.929991011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:43:36.938148883Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:43:37.883683675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:43:38.202108359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44378"
level=debug ts=2026-05-13T21:43:38.738501673Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T21:43:42.031076154Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-13T21:43:42.883917505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:43:43.939054876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:43:43.945866302Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:43:45.22144152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:43:45.228073323Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:43:45.324445022Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-13T21:43:46.00775281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35096"
level=warn ts=2026-05-13T21:43:47.88389659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:43:48.616130444Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-05-13T21:43:49.359166028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:43:50.597137363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:43:50.60169093Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:43:51.909429471Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-13T21:43:52.883180099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:43:55.019479351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:43:55.02711656Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:43:55.202341329Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=warn ts=2026-05-13T21:43:57.883981819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:43:58.494525034Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-05-13T21:44:01.187693569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:44:01.195122188Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:44:01.787828836Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=debug ts=2026-05-13T21:44:01.787852567Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-13T21:44:02.883472961Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:44:02.883501182Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:44:02.883508964Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:44:02.883527443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:44:04.474967568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:44:05.080501617Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-13T21:44:05.832654372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49092"
level=warn ts=2026-05-13T21:44:07.883817182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:44:08.37337433Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=debug ts=2026-05-13T21:44:11.665526841Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-13T21:44:12.883741392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:44:14.957786434Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T21:44:15.385049009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:44:15.386702399Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T21:44:17.883398566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:44:18.250603744Z caller=checkpoint.go:372 msg="writing series" size="427 kB"
level=debug ts=2026-05-13T21:44:18.250627432Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-13T21:44:19.769971654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:44:19.777471694Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:44:21.54400557Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-05-13T21:44:22.692214769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:44:22.700328024Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:44:22.700348576Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:44:22.700379494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:44:22.707677082Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:44:22.883813819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:44:23.219241364Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:44:24.836311139Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-13T21:44:27.872665752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:44:27.878567769Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T21:44:27.883658607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:44:28.131607969Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:44:28.131630618Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=3
level=debug ts=2026-05-13T21:44:31.421655519Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:44:32.864637655Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:44:32.864982277Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-13T21:44:32.883279573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:44:32.883289861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:44:32.88331288Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:44:32.8833202Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:44:34.477603991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:44:34.714613002Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-13T21:44:35.4553249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:44:35.460605761Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:44:35.760299101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:44:35.764470283Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:44:36.087637942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:44:36.095284171Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:44:37.883388804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:44:38.006831516Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
ts=2026-05-13T21:44:38.206046081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34526"
level=debug ts=2026-05-13T21:44:41.30048064Z caller=checkpoint.go:372 msg="writing series" size="417 kB"
level=warn ts=2026-05-13T21:44:42.88407647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:44:44.592072264Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-13T21:44:47.883664249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:44:47.884775258Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=debug ts=2026-05-13T21:44:51.177786119Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-13T21:44:51.177809979Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-13T21:44:51.524756219Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:44:52.883216265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:44:53.663063072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:44:53.674853421Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T21:44:54.338260725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="11 MB" avg_comp="2.1 MB" total_uncomp="93 MB" avg_uncomp="19 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:44:54.346295837Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:44:54.469962361Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-13T21:44:57.762874269Z caller=checkpoint.go:372 msg="writing series" size="821 kB"
level=warn ts=2026-05-13T21:44:57.883368526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:01.055649708Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-13T21:45:01.05567521Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-05-13T21:45:02.883299391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:45:02.883307325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:45:02.883330678Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:45:02.883338735Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:45:04.348244409Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-13T21:45:04.480849922Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T21:45:04.728896861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:45:04.738139438Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T21:45:05.836860421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:35838"
level=debug ts=2026-05-13T21:45:07.64093417Z caller=checkpoint.go:372 msg="writing series" size="485 kB"
level=warn ts=2026-05-13T21:45:07.883973038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:07.938200774Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:45:07.967401462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:45:07.974547964Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T21:45:08.208957863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50482"
level=debug ts=2026-05-13T21:45:10.933859508Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-13T21:45:11.501619578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:45:11.508093888Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T21:45:12.883534844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:14.226729748Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-13T21:45:17.518874956Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:45:17.883695167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:45:18.210057227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:45:18.21535007Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:45:20.812352603Z caller=checkpoint.go:372 msg="writing series" size="474 kB"
level=debug ts=2026-05-13T21:45:20.812374027Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=warn ts=2026-05-13T21:45:22.883206041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:24.104866639Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T21:45:26.489075099Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:45:27.397925845Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=debug ts=2026-05-13T21:45:27.397948376Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-13T21:45:27.883414437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:30.690034288Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T21:45:32.864838052Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:45:32.865251615Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=debug ts=2026-05-13T21:45:32.866471406Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T21:45:32.866509473Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976342 n=1
level=debug ts=2026-05-13T21:45:32.86655184Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976342 n=1
level=info ts=2026-05-13T21:45:32.883531285Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:45:32.88355179Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:45:32.883559314Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:45:32.883583993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:33.982685716Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T21:45:34.456329913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:45:34.461057134Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T21:45:34.484257466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:45:37.274679642Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-05-13T21:45:37.88333648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:45:38.304171033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:45:38.311847557Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:45:40.568678808Z caller=checkpoint.go:372 msg="writing series" size="721 kB"
level=warn ts=2026-05-13T21:45:42.88384263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:43.860921485Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-13T21:45:43.860943979Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-13T21:45:45.350345327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:45:45.357968215Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:45:47.153169627Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=warn ts=2026-05-13T21:45:47.883719254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:50.446258424Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-13T21:45:52.883985808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:53.738316281Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T21:45:56.56250393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:45:56.571653Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:45:57.046114894Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-13T21:45:57.046137146Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
level=warn ts=2026-05-13T21:45:57.883771803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:45:58.169076635Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:45:59.239774523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:45:59.716300682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:45:59.723213212Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:46:00.324046099Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T21:46:02.542185519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:46:02.548045067Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:46:02.883550198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:46:02.883580055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:46:02.883588851Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:46:02.883652191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:03.617080212Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
ts=2026-05-13T21:46:04.487209485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:46:06.90930786Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-13T21:46:07.883750131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:10.201503619Z caller=checkpoint.go:372 msg="writing series" size="322 B"
level=warn ts=2026-05-13T21:46:12.883695475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:13.495865882Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T21:46:13.495889605Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-13T21:46:14.97848486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:46:14.985573601Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:46:16.787153212Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T21:46:17.884213518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:46:19.686450574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:46:19.692526762Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:46:20.080923055Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-13T21:46:20.080944994Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=2
level=warn ts=2026-05-13T21:46:22.883472797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:23.372515039Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:46:25.65062011Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:46:26.665797047Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-13T21:46:27.883453768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:29.958236257Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-13T21:46:32.03383933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:46:32.046777491Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T21:46:32.864824212Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:46:32.865216585Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:46:32.883489098Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:46:32.883508633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:46:32.88351609Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:46:32.883557968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:33.251218889Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
ts=2026-05-13T21:46:34.490241524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T21:46:35.401051873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:46:35.408127618Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:46:36.543575059Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=warn ts=2026-05-13T21:46:37.883700765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:39.836084344Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=info ts=2026-05-13T21:46:40.132148921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:46:40.140097529Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:46:42.883350315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:43.129321375Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-13T21:46:46.36762416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:46:46.375275885Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:46:46.42132257Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-13T21:46:47.884003027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:49.213803523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:46:49.714489956Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-13T21:46:52.884049812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:53.007342938Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-13T21:46:56.299360454Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-13T21:46:56.299383558Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-13T21:46:57.79014709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:46:57.796628968Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T21:46:57.796648544Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:46:57.796671725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:46:57.801638724Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T21:46:57.883732493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:46:59.080357569Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:46:59.592369456Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=info ts=2026-05-13T21:47:00.005807447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:47:00.012032411Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:47:02.883907262Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:47:02.883944529Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:47:02.883956171Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:47:02.884031315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:47:02.885052165Z caller=checkpoint.go:384 msg="flushing series" totalSize="657 kB" series=1
level=info ts=2026-05-13T21:47:02.90352651Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-13T21:47:02.905193584Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.020888457s
ts=2026-05-13T21:47:04.493830765Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:47:05.843788997Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57754"
level=warn ts=2026-05-13T21:47:07.883814813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:47:08.905852299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:47:08.912496353Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T21:47:12.883226303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:47:17.883209519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:47:17.986595674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:47:17.991764089Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:47:20.478794895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:47:20.48654872Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:47:22.883872284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T21:47:27.884211959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:47:29.40751708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:47:29.417425568Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:47:29.41745581Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:47:29.417494426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:47:29.425191342Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:47:31.90170247Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:47:32.864443258Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:47:32.865507356Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T21:47:32.883879528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T21:47:32.883886339Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-13T21:47:32.883891396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:47:32.883896303Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:47:32.883906505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:47:32.88404904Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-13T21:47:32.884057906Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:47:32.884077776Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.726µs success=true
level=debug ts=2026-05-13T21:47:32.88430897Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-13T21:47:33.391825208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:47:33.399353077Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T21:47:34.496686081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:47:36.098844272Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T21:47:37.883240739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:47:39.315883906Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-13T21:47:39.315907423Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=info ts=2026-05-13T21:47:41.228415027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:47:41.234530372Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:47:42.527858182Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-13T21:47:42.8833824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:47:42.924603531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:47:42.929957147Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:47:42.975059783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:47:42.98117541Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:47:43.258387688Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:47:45.273429575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:47:45.280576242Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:47:45.742194152Z caller=checkpoint.go:372 msg="writing series" size="462 kB"
level=warn ts=2026-05-13T21:47:47.883521447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:47:48.958662792Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T21:47:48.958686677Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=3
level=debug ts=2026-05-13T21:47:52.170675253Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-13T21:47:52.883440261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:47:55.437925316Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-13T21:47:55.437956544Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=2
level=warn ts=2026-05-13T21:47:57.883408432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:47:58.5990525Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-13T21:48:01.815284608Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T21:48:01.815308163Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-13T21:48:02.258792469Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:48:02.684974766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:48:02.691520071Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:48:02.883708824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:48:02.883725912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:48:02.883750837Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:48:02.883758034Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:48:04.500128253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:48:05.027687024Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
ts=2026-05-13T21:48:05.847920338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:45030"
level=warn ts=2026-05-13T21:48:07.883525067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:48:08.093751888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:48:08.10768229Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:48:08.107720207Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:48:08.107757112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:48:08.115480575Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:48:08.241917762Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-13T21:48:11.456314543Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T21:48:12.883369878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:48:14.670465514Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T21:48:17.883412079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:48:17.88453924Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T21:48:19.446615799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:48:19.458356418Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:48:21.106510507Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-13T21:48:22.88420074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:48:23.350728635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:48:23.35946922Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:48:24.313523158Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T21:48:27.527985874Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-13T21:48:27.543080073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:48:27.551205872Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:48:27.658695231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T21:48:27.658755634Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T21:48:27.662644132Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T21:48:27.883763563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:48:27.959065537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="156 kB" avg_uncomp="156 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:48:27.962958226Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T21:48:27.979010382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:48:27.986737862Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:48:27.98675583Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:48:28.259537946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:48:28.26338471Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T21:48:28.559283803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="121 B" avg_comp="121 B" total_uncomp="157 kB" avg_uncomp="157 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:48:28.563003382Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T21:48:28.859145218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:48:28.862622123Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T21:48:29.159456792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:48:29.16308958Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T21:48:29.459493409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:48:29.462903543Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T21:48:29.759649051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="101 B" avg_uncomp="101 B" idle=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:48:29.763041567Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:48:30.059060366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="29 kB" avg_uncomp="29 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:48:30.06259863Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T21:48:30.359363446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:48:30.364107287Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T21:48:30.658981316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:48:30.665109572Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:48:30.742237231Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T21:48:31.127445326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:48:31.132160489Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:48:32.864714381Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:48:32.865059455Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:48:32.883373194Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:48:32.883390699Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:48:32.883397871Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:48:32.883391477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:48:33.956038168Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-13T21:48:34.503046775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T21:48:37.170591702Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:48:37.170617046Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-13T21:48:37.883502584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:48:38.221281624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:54372"
level=info ts=2026-05-13T21:48:38.84267147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:48:38.846332339Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T21:48:39.230663145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:48:39.234234992Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T21:48:39.479359002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:48:39.481146995Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T21:48:40.362869979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:48:40.368901433Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:48:40.368922008Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:48:40.368943052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:48:40.374377028Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:48:40.385853346Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T21:48:40.385866885Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
level=info ts=2026-05-13T21:48:40.507929522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:48:40.511505052Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:48:41.748770134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:48:41.752273241Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T21:48:42.485521447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:48:42.489355058Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T21:48:42.884267711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:48:43.599486025Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-13T21:48:44.01967797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:48:44.024751627Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T21:48:44.91482007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:48:44.918281542Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T21:48:45.27333659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:48:45.280841437Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T21:48:45.369061952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:48:45.372301235Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T21:48:45.602874997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:48:45.606453759Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T21:48:46.023150383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41168"
level=debug ts=2026-05-13T21:48:46.813590601Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=warn ts=2026-05-13T21:48:47.883583153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:48:48.329238543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:48:48.332881273Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T21:48:50.028081075Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T21:48:50.05320876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:48:50.060448703Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T21:48:52.883842328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:48:53.242367071Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-13T21:48:54.791844715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:48:54.797589869Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:48:56.456053487Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-13T21:48:57.883786383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:48:58.769069441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:48:58.773165928Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:48:59.670653202Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=debug ts=2026-05-13T21:49:01.279143684Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:49:02.883369956Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:49:02.88340075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:49:02.8834085Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:49:02.883429179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:49:02.884555951Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
ts=2026-05-13T21:49:04.505954491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T21:49:05.028141072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:49:05.03300249Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T21:49:05.221774086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:49:05.225951676Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T21:49:05.852320326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47398"
level=info ts=2026-05-13T21:49:06.024517101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:49:06.028639984Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T21:49:06.099205315Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=debug ts=2026-05-13T21:49:06.099233752Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-13T21:49:07.884005602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:49:09.313341Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T21:49:09.496817693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:49:09.506636953Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:49:09.782525488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:49:09.786549578Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T21:49:11.221734201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="578 B" avg_comp="578 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:49:11.225735494Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T21:49:12.527964475Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=info ts=2026-05-13T21:49:12.725529545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:49:12.735187647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:49:12.88382819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:49:15.743129088Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-13T21:49:15.818289394Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:49:16.026411791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56356"
level=info ts=2026-05-13T21:49:17.701355609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:49:17.705805538Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T21:49:17.883552455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:49:18.956096897Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T21:49:18.96312693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:49:18.966960469Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:49:19.919606658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:49:19.923185116Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T21:49:20.997077062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:49:21.001322779Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T21:49:21.782510806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:49:21.791859478Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:49:21.888614828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:49:21.893311457Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T21:49:22.171436252Z caller=checkpoint.go:372 msg="writing series" size="570 kB"
level=warn ts=2026-05-13T21:49:22.883993198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:49:24.483685471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:49:24.490947776Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:49:25.385489722Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:49:25.38551263Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-13T21:49:25.535836512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:49:25.542999455Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T21:49:27.884154311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:49:28.598730311Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-13T21:49:30.757404206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:49:30.765784422Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:49:31.818910556Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:49:31.81896046Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=info ts=2026-05-13T21:49:32.864623683Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:49:32.86483443Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:49:32.883129523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:49:32.883157766Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:49:32.883165611Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:49:32.883231825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:49:34.509006736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:49:35.028088102Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-13T21:49:35.857084273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50814"
level=debug ts=2026-05-13T21:49:36.576734697Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:49:37.883720818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:49:38.242653922Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=debug ts=2026-05-13T21:49:41.45639615Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-13T21:49:42.883193138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:49:44.159806838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:49:44.167799473Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:49:44.167836869Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:49:44.167872014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:49:44.172275652Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:49:44.671010351Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=warn ts=2026-05-13T21:49:47.88384941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:49:47.885105291Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-13T21:49:47.885125931Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-13T21:49:49.927218315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:49:49.937129647Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:49:51.099495353Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=warn ts=2026-05-13T21:49:52.883955171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:49:53.782571583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:49:53.790530797Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:49:54.313937766Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-13T21:49:55.908982645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:49:55.913364072Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T21:49:57.527426466Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T21:49:57.883755148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:49:58.185948293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:49:58.19026393Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T21:49:59.604314241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:49:59.608276103Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T21:49:59.666604438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:49:59.670157362Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T21:50:00.742714264Z caller=checkpoint.go:372 msg="writing series" size="809 kB"
level=debug ts=2026-05-13T21:50:00.742742027Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T21:50:00.853429219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:50:00.857534937Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T21:50:02.45116781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:50:02.455412169Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:50:02.88332293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:50:02.883329973Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:50:02.883358663Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:50:02.883369499Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:50:03.956115316Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-13T21:50:04.218394206Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T21:50:04.511946247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T21:50:05.933009222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:50:05.936992965Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T21:50:07.170511926Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-13T21:50:07.88359492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:50:08.094363499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:50:08.100121963Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:50:08.585652361Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:50:10.385469202Z caller=checkpoint.go:372 msg="writing series" size="616 kB"
level=info ts=2026-05-13T21:50:11.712978239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:50:11.716738172Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T21:50:11.797804546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:50:11.801509362Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T21:50:12.883907516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:50:13.599433262Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-13T21:50:13.599454302Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T21:50:14.335863601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:50:14.343723125Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T21:50:14.480274294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:50:14.486509544Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:50:14.984966618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:50:14.990104804Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T21:50:16.813525835Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-13T21:50:17.884025579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:50:20.027560062Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T21:50:20.577125817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:50:20.583815829Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T21:50:22.883752818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:50:23.241861264Z caller=checkpoint.go:372 msg="writing series" size="688 kB"
level=info ts=2026-05-13T21:50:23.588834657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:50:23.594967032Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:50:26.456187564Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T21:50:26.456212808Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-13T21:50:27.88430365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:50:29.671554794Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=info ts=2026-05-13T21:50:32.864652641Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:50:32.864703471Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:50:32.884039913Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:50:32.88406209Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:50:32.88407285Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:50:32.88415621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:50:32.885179441Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-13T21:50:32.893483003Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T21:50:34.514992968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
ts=2026-05-13T21:50:35.861637618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42062"
level=debug ts=2026-05-13T21:50:36.099204251Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-13T21:50:37.883852397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:50:38.59245183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:50:38.598754799Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:50:39.314066944Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-13T21:50:39.314089334Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T21:50:40.390817826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:50:40.394568102Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T21:50:41.347146604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:50:41.354650353Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:50:41.354671561Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:50:41.354708908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:50:41.361522523Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:50:42.527937917Z caller=checkpoint.go:372 msg="writing series" size="855 kB"
level=warn ts=2026-05-13T21:50:42.883204311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:50:43.022787106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:50:45.741619681Z caller=checkpoint.go:372 msg="writing series" size="428 kB"
level=debug ts=2026-05-13T21:50:45.741644283Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-13T21:50:47.638689719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:50:47.644087324Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T21:50:47.796532541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:50:47.800419561Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T21:50:47.883966366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:50:48.076854116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:50:48.083820759Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:50:48.956186742Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=info ts=2026-05-13T21:50:50.204659207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:50:50.208038174Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T21:50:52.170490178Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-13T21:50:52.884068452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:50:53.804590384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:50:53.808477049Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T21:50:53.992933224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:50:53.996694625Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T21:50:55.385121629Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T21:50:57.883625691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:50:58.612366018Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-13T21:50:58.61238826Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
level=info ts=2026-05-13T21:50:59.24334811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:50:59.251379367Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:50:59.930225851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:50:59.933396884Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:51:01.498600144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:51:01.502116523Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T21:51:01.813892701Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-13T21:51:02.741592895Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:51:02.883581144Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:51:02.88357067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:51:02.883606073Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:51:02.883614306Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T21:51:04.518130313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:51:05.0280462Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-05-13T21:51:06.263421055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:51:06.269566476Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:51:07.631694926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:51:07.63533222Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T21:51:07.883425254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:51:08.241906372Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T21:51:10.419076194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:51:10.422764386Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T21:51:10.474120375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:51:10.479862619Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:51:10.732891933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:51:10.739950993Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:51:11.456029396Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=warn ts=2026-05-13T21:51:12.883940342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:51:14.671607853Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T21:51:14.671630654Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
ts=2026-05-13T21:51:16.033273823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:43564"
level=info ts=2026-05-13T21:51:16.893594329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:51:16.899736654Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T21:51:17.883489309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:51:17.884578488Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-13T21:51:21.101337057Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T21:51:21.101359101Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=warn ts=2026-05-13T21:51:22.884088017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:51:23.548069991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:51:23.55306096Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:51:24.313517684Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-13T21:51:27.527824992Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-13T21:51:27.883561803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:51:29.127629802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:51:29.201850061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:51:29.205848933Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T21:51:30.741682124Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-13T21:51:31.075811672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:51:31.07941099Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T21:51:32.864771553Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:51:32.865141011Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-13T21:51:32.883425408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:51:32.883438181Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:51:32.883455204Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:51:32.883462088Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:51:33.767555547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:51:33.775480385Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:51:33.953758663Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:51:33.95585946Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-05-13T21:51:34.521089156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:51:37.170843271Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=warn ts=2026-05-13T21:51:37.884044316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:51:38.799467921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:51:38.803337783Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T21:51:39.266992389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:51:39.270456634Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T21:51:40.38505418Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-13T21:51:41.937453513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:51:41.940970019Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T21:51:42.34005367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:51:42.343684923Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-13T21:51:42.88385143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:51:43.599519526Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-13T21:51:46.777852879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:51:46.785690451Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:51:46.813766074Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-13T21:51:47.556060309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:51:47.562166336Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T21:51:47.883453181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:51:49.977792014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:51:49.981493945Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T21:51:50.027615209Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-13T21:51:51.502717106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:51:51.506162459Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T21:51:52.510680422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:51:52.514710762Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T21:51:52.590912203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:51:52.595804322Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T21:51:52.883234091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:51:53.242087357Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-13T21:51:56.456594233Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-13T21:51:57.883835988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:51:58.641122108Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:51:59.671477235Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-13T21:51:59.671500587Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=12
level=info ts=2026-05-13T21:52:02.884152161Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:52:02.884180971Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:52:02.884190544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:52:02.884153744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:52:02.891059201Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-13T21:52:02.892826866Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008588763s
ts=2026-05-13T21:52:04.52412775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T21:52:05.867818021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39328"
level=debug ts=2026-05-13T21:52:07.207657061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:52:07.209610739Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T21:52:07.883915002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:52:08.232661022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41294"
level=info ts=2026-05-13T21:52:09.336272976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:52:09.339952833Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T21:52:10.126227561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:52:10.132808495Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T21:52:10.311490554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:52:10.319813801Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T21:52:10.916474776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:52:10.92076234Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:52:12.883746757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:52:16.037932898Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41300"
level=warn ts=2026-05-13T21:52:17.884209152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:52:19.273373732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:52:19.277701804Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T21:52:21.83169607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:52:21.840205365Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:52:22.883921408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:52:26.757798046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:52:26.761684367Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T21:52:27.883745988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:52:29.801657593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:52:29.80594024Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:52:30.504000033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:52:30.507731061Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T21:52:32.544058827Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:52:32.864450476Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:52:32.86550259Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T21:52:32.883872761Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-13T21:52:32.883879052Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-13T21:52:32.883896786Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T21:52:32.883893284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:52:32.883904814Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T21:52:32.884044198Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-13T21:52:32.884055215Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:52:32.884075272Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.58µs success=true
level=info ts=2026-05-13T21:52:32.884146015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=debug ts=2026-05-13T21:52:32.884390766Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=error ts=2026-05-13T21:52:32.887587252Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T21:52:33.844762088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:52:33.84849372Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T21:52:33.979455493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:52:33.983310426Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-13T21:52:34.527554331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T21:52:34.551026614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:52:34.554525677Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T21:52:36.098931917Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T21:52:37.523065981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:52:37.528242261Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T21:52:37.883748592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:52:39.317293538Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-13T21:52:39.317323171Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=3
level=info ts=2026-05-13T21:52:40.608299054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:52:40.615376629Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:52:42.527996784Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-13T21:52:42.883311461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:52:43.488158531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:52:43.49419352Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:52:45.742006001Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
ts=2026-05-13T21:52:46.040972154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55376"
level=info ts=2026-05-13T21:52:46.087755929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:52:46.091612784Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T21:52:47.883841958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:52:48.957502191Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-13T21:52:48.957524906Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=3
level=debug ts=2026-05-13T21:52:52.170866893Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-13T21:52:52.883235301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:52:55.438629201Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-13T21:52:55.438657234Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=2
level=warn ts=2026-05-13T21:52:57.88385994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:52:57.940918647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:52:57.950443774Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:52:58.599172901Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-13T21:52:59.266862123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:52:59.271486544Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:53:01.207937947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:53:01.21597603Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:53:01.81480263Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T21:53:01.814823957Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-13T21:53:02.883375009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:53:02.883386125Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:53:02.883414213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:53:02.88342366Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:53:04.244068734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:53:04.53015456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:53:05.028128418Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
ts=2026-05-13T21:53:05.87198259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52146"
level=info ts=2026-05-13T21:53:06.162283659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:53:06.16720855Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T21:53:07.884202087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:53:08.095321509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:53:08.241819065Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-13T21:53:11.456323078Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T21:53:12.884098139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:53:14.294217812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:53:14.297919764Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T21:53:14.671063916Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-13T21:53:15.576461339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:53:15.58006698Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T21:53:17.883607837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:53:17.884911424Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T21:53:19.779251179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:53:19.788175155Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:53:19.78917192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:53:19.792442251Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T21:53:21.099437571Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T21:53:21.243709628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:53:21.247097829Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T21:53:21.357571757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:53:21.360740434Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T21:53:21.931166314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:53:21.934475073Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T21:53:22.883261794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:53:23.091076458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:53:23.098453203Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:53:23.997066119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:53:24.009992907Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T21:53:24.010034749Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:53:24.010060404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:53:24.014227972Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:53:24.313216463Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-13T21:53:27.248159934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:53:27.252010004Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T21:53:27.527932272Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T21:53:27.883417754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:53:30.742484011Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T21:53:31.466757944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:53:31.470563467Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T21:53:32.864611069Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:53:32.865400279Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:53:32.88377334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:53:32.883794626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:53:32.883805378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:53:32.883830464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:53:33.955947136Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-13T21:53:34.533159818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T21:53:36.264484165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:53:36.269091396Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T21:53:37.170904293Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:53:37.170925996Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=warn ts=2026-05-13T21:53:37.883813917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:53:38.528367425Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:53:39.022065079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:53:39.027556811Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:53:39.354531556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:53:39.36133204Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:53:40.386669834Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-13T21:53:40.386691597Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=1
level=warn ts=2026-05-13T21:53:42.883257428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:53:42.978870251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:53:42.986753236Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:53:43.599353113Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-13T21:53:44.706829158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:53:44.714773393Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:53:44.89441164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T21:53:46.044604583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:39690"
level=debug ts=2026-05-13T21:53:46.813401954Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=warn ts=2026-05-13T21:53:47.883420575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:53:49.652061734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:53:49.656543938Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:53:50.028071445Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-13T21:53:52.883855296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:53:53.241506355Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-13T21:53:53.68380168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:53:53.687703577Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T21:53:56.456688003Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T21:53:56.592131841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:53:56.596420701Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T21:53:57.883518675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:53:59.671431331Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T21:54:02.883256896Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:54:02.883271332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:54:02.883286627Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:54:02.883295144Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:54:02.884475455Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
ts=2026-05-13T21:54:04.535571505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:54:06.099524516Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=debug ts=2026-05-13T21:54:06.09954751Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-13T21:54:07.497441879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:54:07.504954506Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:54:07.702504128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:54:07.706225877Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T21:54:07.88405305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:54:08.469135623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:54:08.473554844Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T21:54:09.313153766Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T21:54:09.961834485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:54:09.966220067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:54:09.969789843Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:54:09.969788985Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T21:54:11.401678769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:54:11.405750814Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T21:54:11.511168128Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:54:12.352470456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:54:12.360331174Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:54:12.360354867Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:54:12.360383086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:54:12.367540911Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:54:12.528138644Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-13T21:54:12.883839272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:54:12.950004496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:54:12.95527941Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T21:54:15.617539535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:54:15.622626347Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:54:15.742035771Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T21:54:17.883663395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:54:17.916818685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:54:17.920515379Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T21:54:18.956718975Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-13T21:54:22.171136896Z caller=checkpoint.go:372 msg="writing series" size="480 kB"
level=warn ts=2026-05-13T21:54:22.88340612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:54:24.151551486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="578 B" avg_comp="578 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:54:24.155980471Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T21:54:25.38473271Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:54:25.384756227Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-13T21:54:27.746328864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:54:27.754315441Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T21:54:27.883981922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:54:28.599247191Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-13T21:54:31.226144731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:54:31.234164519Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:54:31.816230903Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:54:31.816265037Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=info ts=2026-05-13T21:54:32.864610054Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:54:32.865474346Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:54:32.883810374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:54:32.883831047Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:54:32.883838759Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:54:32.883860323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:54:34.033888043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:54:34.037632161Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-13T21:54:34.53806631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:54:35.027901368Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:54:36.159187615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:54:36.166335697Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:54:36.166357201Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:54:36.166380877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:54:36.173755991Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:54:37.883889837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:54:38.242421681Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-13T21:54:38.372118148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:54:38.376927933Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:54:38.376950386Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:54:38.376971159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:54:38.380185954Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T21:54:39.951631538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:54:39.95721001Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:54:41.456324252Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-13T21:54:42.88319183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:54:44.358698779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:54:44.670848511Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
level=debug ts=2026-05-13T21:54:44.837503913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T21:54:47.883761285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:54:47.88494248Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-13T21:54:49.241647678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:54:49.245807531Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:54:50.446140571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:54:50.45330618Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:54:51.098971154Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-13T21:54:52.444638045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:54:52.452644214Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:54:52.883522045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:54:54.3130526Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-13T21:54:54.313073322Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-13T21:54:54.936948462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:54:54.940449807Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T21:54:56.198945273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:54:56.202793503Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:54:56.202815564Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:54:56.2028365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:54:56.206186983Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T21:54:57.527877813Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-13T21:54:57.846718649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:54:57.851692169Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T21:54:57.851710813Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:54:57.85173062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:54:57.854715225Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T21:54:57.884127413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:54:57.934281058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:54:57.937522767Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:54:57.937539456Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:54:58.15190898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:54:58.155305036Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T21:54:58.371497991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:54:58.374755803Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:54:58.374773104Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:54:58.451988719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:54:58.453748359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:54:58.45523571Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:54:58.455238972Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:54:58.455253687Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:54:58.752805447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:54:58.755919576Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T21:55:00.7427571Z caller=checkpoint.go:372 msg="writing series" size="916 kB"
level=info ts=2026-05-13T21:55:01.589125131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:55:01.592589195Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:55:01.592608089Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:55:01.592629137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:55:01.595433001Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T21:55:02.883958434Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:55:02.88398702Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:55:02.88399675Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:55:02.884000613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:55:03.780390187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:55:03.788067224Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:55:03.956571841Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=debug ts=2026-05-13T21:55:03.956604964Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-13T21:55:04.149137813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:55:04.153924551Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-13T21:55:04.541428486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:55:05.878947333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37354"
level=info ts=2026-05-13T21:55:07.15468364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:55:07.158696318Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:55:07.158714911Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:55:07.158733533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:55:07.16191886Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T21:55:07.171072391Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-13T21:55:07.883428891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:55:10.384604145Z caller=checkpoint.go:372 msg="writing series" size="649 kB"
level=debug ts=2026-05-13T21:55:12.496350019Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T21:55:12.884042632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:55:13.599135583Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-13T21:55:13.735504778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:55:13.739928255Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T21:55:14.087694211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:55:14.091756219Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T21:55:15.0455082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:55:15.053465792Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:55:16.813223993Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-13T21:55:16.813254353Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T21:55:17.263797709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:55:17.267563587Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T21:55:17.473060402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:55:17.476706418Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:55:17.598966583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:55:17.602737701Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:55:17.602758257Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:55:17.602780643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:55:17.60582908Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T21:55:17.883229829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:55:18.11369708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:55:18.117306819Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T21:55:19.122747412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:55:19.126471823Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T21:55:20.027525354Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-13T21:55:22.884094372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:55:23.197599817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:55:23.201750523Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T21:55:23.242095511Z caller=checkpoint.go:372 msg="writing series" size="688 kB"
level=info ts=2026-05-13T21:55:25.873350083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:55:25.879224657Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:55:25.87924461Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:55:25.879267466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:55:25.884919009Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:55:26.456820385Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T21:55:26.456842923Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-05-13T21:55:27.88372128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:55:27.958051959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:55:27.963155969Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T21:55:29.671746529Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=info ts=2026-05-13T21:55:29.971057974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:55:29.974298073Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:55:29.974319582Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:55:29.97434275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:55:29.976912133Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T21:55:32.864655607Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:55:32.864674813Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:55:32.884044437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T21:55:32.884043013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:55:32.884066649Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:55:32.884074684Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:55:32.885240983Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T21:55:33.825438342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:55:33.828634799Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T21:55:33.970046552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:55:33.973457459Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T21:55:34.544191739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:55:35.548009259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:55:35.972545084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:55:35.978282976Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:55:36.09876652Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T21:55:37.221303209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:55:37.227004083Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:55:37.227041106Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:55:37.227063878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:55:37.232316594Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T21:55:37.883485243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:55:38.243070501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44400"
level=info ts=2026-05-13T21:55:39.147523288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:55:39.150990391Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T21:55:39.313931263Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-13T21:55:39.31395401Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T21:55:40.34473065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:55:40.348422318Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T21:55:40.649151122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:55:40.652683038Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T21:55:42.528426267Z caller=checkpoint.go:372 msg="writing series" size="868 kB"
level=warn ts=2026-05-13T21:55:42.88350646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:55:45.12031089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:55:45.130622187Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:55:45.742646502Z caller=checkpoint.go:372 msg="writing series" size="428 kB"
level=debug ts=2026-05-13T21:55:45.742672942Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
ts=2026-05-13T21:55:46.051600963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:50334"
level=warn ts=2026-05-13T21:55:47.883807691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:55:48.104048773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:55:48.678211293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:55:48.681856541Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T21:55:48.956068537Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-13T21:55:49.205165619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:55:49.209703582Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T21:55:49.572356995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:55:49.576491357Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T21:55:49.583534447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:55:49.588688033Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:55:50.644422772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:55:50.64798472Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T21:55:52.170310637Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-13T21:55:52.883853492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:55:55.384675929Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-13T21:55:55.497068718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:55:55.504739175Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:55:55.878974318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:55:55.882724652Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T21:55:57.201319396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:55:57.20510404Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T21:55:57.883361337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:55:58.613402766Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-13T21:55:58.613422333Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=4
level=debug ts=2026-05-13T21:56:01.813182239Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T21:56:02.883132109Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:56:02.883163747Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:56:02.883171784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:56:02.883188839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:56:03.561067591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:56:03.566860624Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-13T21:56:04.546866966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:56:05.027655348Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
ts=2026-05-13T21:56:05.883736416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60656"
level=warn ts=2026-05-13T21:56:07.884207962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:56:08.241961391Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-13T21:56:08.246181496Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59734"
level=debug ts=2026-05-13T21:56:10.107291555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T21:56:10.109173728Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T21:56:11.416496691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:56:11.420222069Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T21:56:11.456479869Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-13T21:56:12.736980755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:56:12.743466711Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T21:56:12.883554513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:56:14.672431086Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T21:56:14.672451243Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-13T21:56:14.843241104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:56:14.851472708Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:56:17.883550298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:56:17.884600075Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-13T21:56:19.965622465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:56:19.968940606Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:56:20.301938975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:56:20.306037926Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:56:20.306058097Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:56:20.306080879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:56:20.309241278Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:56:21.039277109Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:56:21.100651152Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T21:56:21.100674967Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=info ts=2026-05-13T21:56:22.76111837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:56:22.764721943Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T21:56:22.883455499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:56:24.31394391Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-13T21:56:26.945157512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:56:26.949213603Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T21:56:27.441453021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:56:27.447868405Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T21:56:27.528215016Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T21:56:27.778004372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:56:27.786880271Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:56:27.883475006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:56:30.741827065Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=info ts=2026-05-13T21:56:32.864631464Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:56:32.865516675Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:56:32.88414739Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:56:32.884166979Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:56:32.884175216Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:56:32.884208257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:56:33.061001901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:56:33.064649087Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T21:56:33.956073412Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-05-13T21:56:34.469416771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:56:34.475077751Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-13T21:56:34.55024406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T21:56:36.465122363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:56:36.468585695Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T21:56:37.170784646Z caller=checkpoint.go:372 msg="writing series" size="350 kB"
level=warn ts=2026-05-13T21:56:37.883954648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T21:56:38.249531868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:41260"
level=info ts=2026-05-13T21:56:38.533401066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:56:38.540532188Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T21:56:39.14332525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:56:39.147173793Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T21:56:40.38496734Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T21:56:42.883535643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:56:43.599334537Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-13T21:56:46.470365086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:56:46.474741702Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T21:56:46.81336546Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=debug ts=2026-05-13T21:56:47.366237341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:56:47.412278243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:56:47.416088517Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T21:56:47.88389886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:56:50.028136922Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-13T21:56:52.883379421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:56:53.224513308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:56:53.241589549Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-13T21:56:54.303564746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:56:54.308443422Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T21:56:56.168716784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:56:56.172445413Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T21:56:56.456226311Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-13T21:56:56.456247395Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-13T21:56:57.883596591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:56:58.191142503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:56:58.195147188Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T21:56:58.592137877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:56:58.600035288Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:56:59.671595261Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=info ts=2026-05-13T21:57:02.883826437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:57:02.883854346Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:57:02.883862231Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:57:02.883875244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:57:02.88490551Z caller=checkpoint.go:384 msg="flushing series" totalSize="657 kB" series=1
level=info ts=2026-05-13T21:57:02.89134329Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-13T21:57:02.892940845Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00869685s
ts=2026-05-13T21:57:04.553571298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:57:05.888514172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:46478"
level=warn ts=2026-05-13T21:57:07.883399338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:57:10.048593713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:57:10.057371544Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T21:57:12.193301791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:57:12.197733452Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T21:57:12.883202378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:57:13.662411686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:57:13.666342435Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T21:57:15.65713491Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:57:16.857270964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:57:16.861101591Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T21:57:17.16697108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:57:17.171038595Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-13T21:57:17.883544368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:57:18.414483802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:57:18.418116272Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T21:57:19.506439478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:57:19.510332261Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T21:57:22.079281688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:57:22.08742047Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:57:22.087442289Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T21:57:22.087476549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:57:22.094477511Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:57:22.883593343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:57:23.115292887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:57:23.121710932Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:57:24.529135508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:57:24.535882144Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:57:27.883456465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:57:28.931067187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:57:28.935308666Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T21:57:30.695954098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:57:30.69962431Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T21:57:32.618568034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:57:32.622762784Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T21:57:32.864688421Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:57:32.864872701Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:57:32.883095699Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T21:57:32.883135922Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:57:32.883155863Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:57:32.883164534Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:57:32.883173689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:57:32.883233079Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-13T21:57:32.883241433Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T21:57:32.883258554Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.648µs success=true
level=debug ts=2026-05-13T21:57:32.883548312Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T21:57:34.556940688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T21:57:36.098298163Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T21:57:37.883940073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:57:37.997249368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:57:38.004372153Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:57:38.931010766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:57:39.31514573Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-13T21:57:39.315175602Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=3
level=info ts=2026-05-13T21:57:41.428541014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:57:41.436394908Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:57:42.527046474Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-13T21:57:42.883424212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:57:44.557310168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:57:44.560865984Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T21:57:45.741581731Z caller=checkpoint.go:372 msg="writing series" size="640 kB"
level=warn ts=2026-05-13T21:57:47.883592756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:57:48.957763711Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-13T21:57:48.957785829Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=debug ts=2026-05-13T21:57:50.424973958Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:57:51.233148116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:57:51.242734009Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:57:52.169771672Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=info ts=2026-05-13T21:57:52.497673668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:57:52.503772346Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T21:57:52.884077338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:57:55.434166782Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-13T21:57:55.43419949Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=2
level=info ts=2026-05-13T21:57:57.042518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:57:57.047780171Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T21:57:57.883426924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:57:58.59819674Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-13T21:58:01.657692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:58:01.66585078Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:58:01.813974824Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T21:58:01.814030766Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T21:58:02.530869222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:58:02.535316833Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:58:02.883861632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:58:02.883891391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:58:02.883900518Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:58:02.883943589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:58:03.855297155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:58:03.861131527Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T21:58:04.55956639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T21:58:04.939126405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:58:04.942635128Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T21:58:05.027038539Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-13T21:58:07.884111082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:58:08.240816499Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-13T21:58:10.508478513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:58:10.512450142Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T21:58:11.454981126Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-13T21:58:11.47118167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:58:11.475035185Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:58:12.658589257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:58:12.662049059Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T21:58:12.883184785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:58:13.393607207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:58:13.397555165Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T21:58:13.629134284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:58:13.632546149Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T21:58:14.524989975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:58:14.52863331Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T21:58:14.670229148Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T21:58:17.88378393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:58:17.884001975Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-13T21:58:20.100801628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:58:21.09850483Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-13T21:58:22.883749545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:58:23.446955076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:58:23.455850243Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:58:24.312967278Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-13T21:58:26.349329535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T21:58:27.526684137Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-13T21:58:27.811805386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:58:27.815951349Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T21:58:27.883548176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:58:28.037853246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:58:28.041415659Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T21:58:30.741217255Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T21:58:32.864654472Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:58:32.864676162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:58:32.884037629Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:58:32.884055394Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:58:32.88406481Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:58:32.884065761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:58:33.955403996Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-13T21:58:34.562201295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T21:58:36.888334909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:58:36.894274507Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:58:37.170066466Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T21:58:37.170092016Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=warn ts=2026-05-13T21:58:37.884131315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:58:40.386471118Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T21:58:40.386492496Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=info ts=2026-05-13T21:58:41.037711975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:58:41.044165521Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T21:58:42.240350607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:58:42.255116902Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T21:58:42.883502503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:58:43.59826416Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-13T21:58:46.812381827Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=warn ts=2026-05-13T21:58:47.88369813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:58:48.194039305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:58:48.198113297Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T21:58:50.02680979Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T21:58:52.068128182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:58:52.072097109Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T21:58:52.237484759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T21:58:52.419976258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:58:52.423764008Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T21:58:52.883920406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:58:53.240627119Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-13T21:58:54.237523678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:58:54.246575158Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T21:58:56.455701032Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T21:58:57.061641418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:58:57.065241109Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T21:58:57.883867877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:58:59.670194157Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T21:58:59.752658014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:58:59.758749916Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:59:00.032450891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:59:00.03888341Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T21:59:01.443730333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:59:01.447678261Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T21:59:01.990365803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:59:01.994035521Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T21:59:02.883894582Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:59:02.883923476Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T21:59:02.883912256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:59:02.883932009Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T21:59:02.884086469Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
ts=2026-05-13T21:59:04.565007404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T21:59:05.896321233Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37906"
level=debug ts=2026-05-13T21:59:06.098543354Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=debug ts=2026-05-13T21:59:06.098567641Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-05-13T21:59:06.923848902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T21:59:07.392095533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:59:07.395816054Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-13T21:59:07.883617825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:59:09.313226296Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T21:59:11.554486457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T21:59:11.558658824Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T21:59:12.27463477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:59:12.283375022Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T21:59:12.527109049Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=warn ts=2026-05-13T21:59:12.883626247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:59:13.976946645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T21:59:13.980745697Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T21:59:15.741545976Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T21:59:17.884135865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:59:18.955277488Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-13T21:59:19.054466649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="578 B" avg_comp="578 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T21:59:19.058952473Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T21:59:19.166461993Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:59:22.169879809Z caller=checkpoint.go:372 msg="writing series" size="616 kB"
level=warn ts=2026-05-13T21:59:22.883246508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:59:23.149140748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T21:59:23.157523371Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T21:59:25.384224315Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-13T21:59:25.384244023Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-13T21:59:27.883323362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:59:28.598599865Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-13T21:59:30.872151297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T21:59:30.880814625Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T21:59:31.815549897Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T21:59:31.815574655Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=2
level=info ts=2026-05-13T21:59:32.864663152Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T21:59:32.86484572Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T21:59:32.883248297Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T21:59:32.88326821Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T21:59:32.883276649Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T21:59:32.883284188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:59:34.501219218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T21:59:34.504942456Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-13T21:59:34.568259077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T21:59:35.02708427Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T21:59:37.182305827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T21:59:37.189925664Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T21:59:37.884087333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:59:38.240871078Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
ts=2026-05-13T21:59:38.260128478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48756"
level=info ts=2026-05-13T21:59:38.68394267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T21:59:38.68820166Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T21:59:40.842494253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T21:59:40.848575581Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T21:59:41.455237715Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-13T21:59:41.81115227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T21:59:41.815144268Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T21:59:42.540136795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T21:59:42.547384758Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T21:59:42.884212835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T21:59:44.111186878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T21:59:44.114699148Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T21:59:44.196001523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T21:59:44.67071289Z caller=checkpoint.go:372 msg="writing series" size="589 kB"
level=info ts=2026-05-13T21:59:47.6040662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T21:59:47.607701949Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T21:59:47.883640951Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-13T21:59:47.883635506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:59:47.883658811Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-13T21:59:48.25407045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T21:59:48.25774275Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T21:59:50.896127272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T21:59:50.899757456Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T21:59:51.098319399Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-13T21:59:52.560345675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T21:59:52.567414436Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T21:59:52.884082329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T21:59:54.3129319Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-13T21:59:55.790933212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T21:59:55.794735441Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T21:59:57.526865039Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T21:59:57.883516498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:00:00.741749771Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T22:00:00.741777651Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-13T22:00:02.883402625Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:00:02.883430248Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:00:02.88343824Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:00:02.883426984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:00:03.263448301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:00:03.266723005Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:00:03.879960656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:00:03.883775881Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:00:03.956235856Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
ts=2026-05-13T22:00:04.571913283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:00:05.139227544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:00:05.141595621Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T22:00:07.16944615Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-13T22:00:07.883709527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:00:08.263134911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37848"
level=info ts=2026-05-13T22:00:08.63449953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:00:08.642034433Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:00:10.384790152Z caller=checkpoint.go:372 msg="writing series" size="646 kB"
level=debug ts=2026-05-13T22:00:10.384829888Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-13T22:00:12.319722973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:00:12.323670785Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T22:00:12.88385132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:00:13.598315978Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-13T22:00:15.124922936Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:00:15.980465397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:00:15.985743428Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:00:16.812999549Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=warn ts=2026-05-13T22:00:17.883215804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:00:20.026788102Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T22:00:21.94498869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:00:21.949443332Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T22:00:22.883368211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:00:23.241385678Z caller=checkpoint.go:372 msg="writing series" size="688 kB"
level=debug ts=2026-05-13T22:00:23.241412314Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T22:00:24.921220937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:00:24.926813261Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T22:00:24.926834934Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:00:24.926857797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:00:24.93107163Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T22:00:26.455440346Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=warn ts=2026-05-13T22:00:27.883664129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:00:29.67049293Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=debug ts=2026-05-13T22:00:29.670514729Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T22:00:31.731906602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:00:31.737748541Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T22:00:32.864662254Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:00:32.864980007Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=debug ts=2026-05-13T22:00:32.866251974Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T22:00:32.866297097Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976343 n=1
level=debug ts=2026-05-13T22:00:32.866336503Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976343 n=1
level=info ts=2026-05-13T22:00:32.88328481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:00:32.883290259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:00:32.88330417Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:00:32.883311876Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T22:00:32.884443437Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-13T22:00:33.497127985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:00:33.501644626Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:00:33.930481665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:00:33.936199971Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T22:00:34.575395194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:00:35.495212948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:00:35.502724455Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:00:36.098604308Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-13T22:00:37.883863541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:00:38.165479482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:00:38.170612988Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T22:00:38.410060776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:00:38.413559533Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T22:00:39.312381462Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-13T22:00:39.616137747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:00:39.619669477Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:00:42.527307864Z caller=checkpoint.go:372 msg="writing series" size="883 kB"
level=debug ts=2026-05-13T22:00:42.527331478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-13T22:00:42.883390551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:00:42.990772277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:00:42.998376269Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:00:42.998397019Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:00:42.998420849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:00:43.004922533Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:00:44.431232814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:00:44.434733448Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:00:45.430008337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T22:00:45.434154584Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:00:45.434197877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:00:45.437215848Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:00:45.741376324Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=info ts=2026-05-13T22:00:46.757116741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:00:46.761784312Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:00:47.546864973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:00:47.550333383Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T22:00:47.883722906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:00:48.955838759Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-05-13T22:00:52.169980739Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-13T22:00:52.492939644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:00:52.497054541Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:00:52.538065632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:00:52.541461822Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:00:52.883888751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:00:54.398464518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:00:54.401892664Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:00:55.384414155Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T22:00:57.883677378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:00:58.614928241Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-13T22:00:58.614951062Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=5
level=info ts=2026-05-13T22:01:00.08626482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:01:00.091277013Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T22:01:00.434066212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:01:00.437687283Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:01:01.812610587Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T22:01:02.824638135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:01:02.8314949Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:01:02.883130379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:01:02.883152759Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:01:02.883161496Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:01:02.883196293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:01:04.578735458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T22:01:05.026576975Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=info ts=2026-05-13T22:01:05.716455118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:01:05.724730055Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:01:05.724751208Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:05.724772758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:01:05.73182827Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:01:06.038710764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:01:06.042627071Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T22:01:07.883728488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:01:08.241098964Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-13T22:01:11.455244548Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=debug ts=2026-05-13T22:01:12.479312057Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:01:12.479365348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="12 kB" avg_uncomp="12 kB" idle=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:01:12.482912545Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=warn ts=2026-05-13T22:01:12.883855261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:01:13.240064764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:01:13.243519757Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:01:13.243539288Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:13.243558965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:01:13.246347643Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:01:14.670567624Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T22:01:14.670592024Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=debug ts=2026-05-13T22:01:17.883739996Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T22:01:17.883775026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:01:21.100248436Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T22:01:21.100270412Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=info ts=2026-05-13T22:01:21.363417951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:01:21.372177073Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T22:01:22.884124298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:01:24.312974036Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-13T22:01:24.97497864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:01:24.981799587Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:01:25.359868441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:01:25.368064682Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:01:26.127412787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:01:26.130914577Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=debug ts=2026-05-13T22:01:27.526649987Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T22:01:27.609872024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:01:27.6135517Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T22:01:27.88397662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:01:27.907545504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:01:29.580825743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:01:29.587247284Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T22:01:30.74157858Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-13T22:01:31.351337547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:01:31.355031763Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:01:31.355050266Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:31.355070307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:01:31.357834193Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:01:31.68574944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="13 MB" avg_comp="2.2 MB" total_uncomp="116 MB" avg_uncomp="19 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:01:31.693359463Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:01:32.864410393Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:01:32.865462956Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T22:01:32.883826163Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:01:32.883842952Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:01:32.88385094Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:01:32.883896782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:01:33.955893403Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-05-13T22:01:34.58174229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:01:35.652461557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:01:35.65666248Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:01:35.656681096Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:35.65670092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:01:35.65948053Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-13T22:01:35.904743719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52650"
level=info ts=2026-05-13T22:01:36.589624693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:01:36.59324534Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:01:36.875266352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:01:36.878802212Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:01:36.878823039Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:36.878842985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:01:36.881571174Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:01:37.169572779Z caller=checkpoint.go:372 msg="writing series" size="425 kB"
level=warn ts=2026-05-13T22:01:37.883801033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:01:37.969266351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:01:37.972694697Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T22:01:37.972714219Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:37.972736263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:01:37.975628593Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T22:01:38.080384347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:01:38.083596887Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:01:38.083615741Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:38.273603534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:01:38.277073277Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:01:38.36560736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:01:38.369086925Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:01:38.369103077Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:38.401197921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:01:38.4042622Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:01:38.572916614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:01:38.576316779Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:01:40.383937215Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-13T22:01:42.883924446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:01:43.598009042Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-13T22:01:46.36011629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:01:46.363596111Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:01:46.363616404Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:46.363636316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:01:46.366588577Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T22:01:46.812844077Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=debug ts=2026-05-13T22:01:47.060891151Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:01:47.884111066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:01:48.200904166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:01:48.210544178Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:01:49.653988819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:01:49.658419285Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:01:49.658438244Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:01:49.658459543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:01:49.661539162Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:01:50.02646115Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-13T22:01:50.33947348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:01:50.343749628Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=info ts=2026-05-13T22:01:50.45206663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:01:50.455820012Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:01:50.679426405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:01:50.683190771Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:01:51.334624171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:01:51.338525459Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T22:01:52.883821815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:01:53.241169571Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-13T22:01:53.581129019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:01:53.584960944Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:01:54.736298284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:01:54.740318189Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:01:56.455318244Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-13T22:01:57.859318957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:01:57.864315813Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-13T22:01:57.883579133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:01:58.479985506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:01:58.483872284Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:01:59.670237215Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-13T22:01:59.670259025Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=12
level=info ts=2026-05-13T22:02:00.430689814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:02:00.435313219Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:02:01.225518571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:02:01.22936255Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:02:02.561294993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:02:02.56470846Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:02:02.884142794Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:02:02.88417756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:02:02.884185345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:02:02.884203121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:02:02.889996549Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-13T22:02:02.891195616Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007803797s
ts=2026-05-13T22:02:04.58494834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T22:02:05.498961996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:02:05.502454482Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-13T22:02:05.908146883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36882"
level=warn ts=2026-05-13T22:02:07.884215558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:02:10.018430473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:02:12.3904105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:02:12.394153665Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T22:02:12.884122247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:02:13.113372807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:02:13.120666574Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:02:16.847074564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:02:16.852775153Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:02:16.852798812Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:02:16.852822952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:02:16.857984002Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:02:17.785585311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:02:17.883398874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:02:19.514147533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:02:19.518457245Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:02:19.518476102Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:02:19.518498101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:02:19.522242666Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:02:20.266710635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:02:20.272641795Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:02:20.272662636Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:02:20.272685754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:02:20.278721864Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T22:02:22.883526914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:02:23.897225272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:02:23.904412408Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:02:24.319677533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:02:24.323340004Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:02:24.411556781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:02:24.415135347Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:02:26.345472876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:02:26.349430527Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-13T22:02:27.883616659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:02:28.071467273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:02:28.075152464Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:02:28.420533903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:02:28.424189059Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:02:28.895509676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:02:28.899318127Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:02:29.107008168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:02:29.110767479Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:02:32.864301739Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:02:32.865352196Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T22:02:32.883711963Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:02:32.883727581Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:02:32.883735584Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T22:02:32.883741559Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-13T22:02:32.883785009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:02:32.88395683Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-13T22:02:32.883965687Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T22:02:32.883985434Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.117µs success=true
level=debug ts=2026-05-13T22:02:32.884306159Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T22:02:33.583820993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:02:33.587384529Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T22:02:33.786837064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:02:33.792204675Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-13T22:02:34.588037508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T22:02:35.217069439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:02:35.220358333Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-13T22:02:35.911261754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60488"
level=debug ts=2026-05-13T22:02:36.099173835Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T22:02:37.88405734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:02:38.839680828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:02:38.846378618Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T22:02:39.317279189Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-13T22:02:39.317301789Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=3
level=info ts=2026-05-13T22:02:40.43578031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:02:40.439371599Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=debug ts=2026-05-13T22:02:42.527456611Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-13T22:02:42.884083564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:02:44.617305384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:02:44.625400799Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:02:44.877447156Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:02:45.092046811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:02:45.096936296Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T22:02:45.741871185Z caller=checkpoint.go:372 msg="writing series" size="496 kB"
level=warn ts=2026-05-13T22:02:47.884045947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:02:48.95738169Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-13T22:02:48.957402793Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=info ts=2026-05-13T22:02:50.490042702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:02:50.496094419Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:02:50.496121695Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:02:50.496148452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:02:50.500006999Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:02:52.170733736Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-13T22:02:52.883521847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:02:55.436064153Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-13T22:02:55.436095528Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=2
level=info ts=2026-05-13T22:02:55.628055484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:02:55.631941742Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:02:56.900415871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:02:56.904141476Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:02:57.18777486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T22:02:57.884242615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:02:58.598982566Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-13T22:03:01.339788763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:03:01.346293079Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:03:01.648636422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:03:01.659226376Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T22:03:01.814704194Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T22:03:01.814727023Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T22:03:02.091488367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:03:02.095300203Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:03:02.883784754Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:03:02.883812356Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:03:02.883820346Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:03:02.883828596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:03:04.591211144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:03:05.027968817Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-13T22:03:06.695943049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:03:06.704540541Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:03:06.988395003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:03:06.991892287Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T22:03:07.884077372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:03:08.242533155Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
ts=2026-05-13T22:03:08.273988189Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35632"
level=debug ts=2026-05-13T22:03:09.151351932Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:03:10.845063574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:03:10.851860425Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:03:11.153901818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:03:11.157940065Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:03:11.455652316Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-13T22:03:12.019587045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:03:12.024154973Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:03:12.49566004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:03:12.499587426Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T22:03:12.883211982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:03:14.67032088Z caller=checkpoint.go:372 msg="writing series" size="658 B"
ts=2026-05-13T22:03:16.073742149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:56850"
level=warn ts=2026-05-13T22:03:17.884045821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:03:17.885277424Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-13T22:03:21.099196432Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-13T22:03:21.485254655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:03:21.489956247Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:03:22.722144889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:03:22.730410275Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T22:03:22.884130694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:03:24.313861127Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-13T22:03:25.440069096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:03:25.443567334Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T22:03:27.274212324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:03:27.278048925Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:03:27.527699446Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T22:03:27.884147061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:03:28.183625803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:03:28.187549805Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:03:28.319062085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:03:28.322424347Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:03:29.539056477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:03:29.54284902Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=info ts=2026-05-13T22:03:30.336009697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:03:30.344880583Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:03:30.742166482Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T22:03:32.864736466Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:03:32.864738704Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:03:32.88410394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:03:32.884119801Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:03:32.884127854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:03:32.884159465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:03:33.956636858Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-13T22:03:34.298331144Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:03:34.300125195Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-13T22:03:34.594155559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:03:35.914880647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57052"
level=debug ts=2026-05-13T22:03:35.935581374Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:03:36.580127269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:03:36.583862086Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:03:37.170714595Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T22:03:37.170736147Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=warn ts=2026-05-13T22:03:37.883723113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:03:40.386909354Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T22:03:40.386931758Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=warn ts=2026-05-13T22:03:42.883400518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:03:43.598545146Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-13T22:03:44.874940078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:03:44.881310427Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T22:03:45.263357351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:03:45.269925968Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:03:46.813171164Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=warn ts=2026-05-13T22:03:47.883679122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:03:48.930854981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:03:48.93514434Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:03:50.027678126Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T22:03:50.233327524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:03:50.239956645Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T22:03:52.883386494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:03:53.241870612Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=debug ts=2026-05-13T22:03:56.456428062Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T22:03:57.406064795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:03:57.410210499Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T22:03:57.88394655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:03:58.573063084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:03:58.577210057Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:03:59.105289268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:03:59.114111123Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:03:59.114133769Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:03:59.114171514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:03:59.124455328Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:03:59.670952457Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T22:04:00.616816504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:04:00.620749418Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:04:00.682051967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:04:00.685443715Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:04:00.956485794Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:04:02.883795322Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:04:02.883823499Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:04:02.883833369Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:04:02.883852422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:04:02.885072513Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=debug ts=2026-05-13T22:04:02.885083807Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
ts=2026-05-13T22:04:04.597460592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:04:06.099204866Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-13T22:04:07.883503812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:04:08.278211491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47542"
level=info ts=2026-05-13T22:04:08.410858982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:04:08.414723298Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:04:09.31391098Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T22:04:10.30054085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:04:10.30463531Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:04:10.387865546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="578 B" avg_comp="578 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:04:10.391450259Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:04:10.616037094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:04:10.619780201Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:04:11.292871446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:04:11.297281467Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:04:12.527681451Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-13T22:04:12.883636062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:04:13.450279909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:04:13.457753895Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:04:15.499994124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:04:15.508902989Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:04:15.74153837Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T22:04:17.88369394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:04:18.956477467Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-13T22:04:22.170688867Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-13T22:04:22.17071249Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-13T22:04:22.598697024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:04:22.606249846Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:04:22.718641934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:04:22.722345362Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:04:22.883626918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:04:23.789838983Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T22:04:25.385292605Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-13T22:04:27.883389162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:04:28.458908579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:04:28.462863308Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=debug ts=2026-05-13T22:04:28.598851256Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-13T22:04:29.926786353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:04:29.933060335Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:04:31.816454898Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T22:04:31.816478358Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=info ts=2026-05-13T22:04:32.744054108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:04:32.750482084Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T22:04:32.864722488Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:04:32.865096256Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:04:32.883443282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:04:32.883465256Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:04:32.883475007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:04:32.883486448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:04:34.149645807Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T22:04:34.60075457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:04:35.02752481Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T22:04:36.530455192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:04:36.536158764Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T22:04:36.579369875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:04:36.582830719Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T22:04:37.883729868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:04:38.242194754Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
ts=2026-05-13T22:04:38.281107268Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58878"
level=info ts=2026-05-13T22:04:40.901253731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:04:40.904917014Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:04:41.456301038Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-13T22:04:42.688526408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:04:42.692435624Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T22:04:42.883828934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:04:44.67056314Z caller=checkpoint.go:372 msg="writing series" size="491 kB"
ts=2026-05-13T22:04:46.079306936Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49544"
level=info ts=2026-05-13T22:04:46.844335952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:04:46.853150868Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T22:04:47.883249169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:04:47.88440223Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=debug ts=2026-05-13T22:04:47.884418295Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-13T22:04:50.404665025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:04:50.413007115Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:04:51.099071923Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=warn ts=2026-05-13T22:04:52.884040769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:04:53.141765043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:04:53.145321853Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:04:54.313171094Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-13T22:04:55.318072885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:04:55.325588619Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:04:56.276437169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:04:56.280128625Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:04:57.254540734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:04:57.25813588Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:04:57.527910062Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T22:04:57.883378665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:05:00.742254848Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-13T22:05:00.742278761Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-05-13T22:05:02.883374483Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:05:02.883402014Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:05:02.883411259Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:05:02.883401046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:05:03.060297566Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:05:03.204555229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:05:03.208536797Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T22:05:03.956095863Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
ts=2026-05-13T22:05:04.603533243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:05:05.788296792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:05:05.792055073Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:05:06.382064078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:05:06.386126064Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:05:07.170236865Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-13T22:05:07.870501541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:05:07.873929538Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-13T22:05:07.884039892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:05:09.358063937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:05:09.362460417Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:05:10.38536446Z caller=checkpoint.go:372 msg="writing series" size="653 kB"
level=info ts=2026-05-13T22:05:12.527237302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:05:12.530989436Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=warn ts=2026-05-13T22:05:12.883650514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:05:13.599637284Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-13T22:05:13.599661831Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-13T22:05:16.813836902Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=debug ts=2026-05-13T22:05:17.152387736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T22:05:17.883706146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:05:18.656128332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:05:18.659590096Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:05:20.027852984Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T22:05:22.560967109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:05:22.568548611Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T22:05:22.884119116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:05:23.241992273Z caller=checkpoint.go:372 msg="writing series" size="688 kB"
level=info ts=2026-05-13T22:05:24.107726758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:05:24.11324471Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T22:05:25.095749139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:05:25.10145813Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:05:25.31800798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:05:25.321397687Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:05:26.455916985Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T22:05:26.455940338Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-13T22:05:27.884059445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:05:29.671075199Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=info ts=2026-05-13T22:05:29.723606321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:05:29.727670416Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:05:30.026558863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:05:30.031632559Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-13T22:05:32.864671366Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:05:32.864681193Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:05:32.884050734Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:05:32.884068592Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:05:32.884075377Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:05:32.884079216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:05:32.885196827Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=debug ts=2026-05-13T22:05:33.720307267Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T22:05:34.606781981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:05:35.92274457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38602"
level=debug ts=2026-05-13T22:05:36.099064793Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-13T22:05:37.883883121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:05:39.313437371Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-13T22:05:39.313460984Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T22:05:40.120143612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:05:40.124324706Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:05:40.904533066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:05:40.912025562Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:05:41.512626185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:05:41.516510358Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:05:42.527553725Z caller=checkpoint.go:372 msg="writing series" size="682 kB"
level=warn ts=2026-05-13T22:05:42.883899297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:05:45.742364952Z caller=checkpoint.go:372 msg="writing series" size="451 kB"
level=debug ts=2026-05-13T22:05:45.742386858Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-13T22:05:47.883722585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:05:48.95643066Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=debug ts=2026-05-13T22:05:52.170718884Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-13T22:05:52.883790264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:05:54.037672351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:05:54.046295301Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:05:54.552762176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:05:54.556847799Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T22:05:54.617152991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:05:55.178957291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:05:55.182210968Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:05:55.384778723Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T22:05:57.883402798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:05:57.931596484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:05:57.934914341Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=debug ts=2026-05-13T22:05:58.616745366Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-13T22:05:58.616769788Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=4
level=info ts=2026-05-13T22:05:59.38621072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:05:59.390113142Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T22:05:59.445136896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:05:59.448755855Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:06:00.327378006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:06:00.331148809Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T22:06:01.813195297Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T22:06:02.05706065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:06:02.060728006Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:06:02.187988749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:06:02.191539645Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:06:02.883183218Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:06:02.883210425Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T22:06:02.883205634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:06:02.883219479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T22:06:03.519066486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:06:03.527251725Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:06:03.528227282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:06:03.531904781Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T22:06:04.610191554Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:06:05.027932147Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=debug ts=2026-05-13T22:06:06.031234167Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:06:07.797032192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:06:07.800723749Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T22:06:07.883575742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:06:08.242206412Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-13T22:06:08.286407827Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60418"
level=debug ts=2026-05-13T22:06:11.455874937Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-13T22:06:11.802851085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:06:11.809074482Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T22:06:12.88388565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:06:14.627661879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:06:14.634612612Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:06:14.671808548Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T22:06:14.671824042Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=warn ts=2026-05-13T22:06:17.883626316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:06:17.8846629Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-13T22:06:20.020001625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:06:20.026906262Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:06:20.509260988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:06:20.514347644Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-13T22:06:21.101528218Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T22:06:21.101552218Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=info ts=2026-05-13T22:06:21.303740218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:06:21.311849697Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T22:06:22.883608248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:06:24.313045901Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=debug ts=2026-05-13T22:06:27.528073644Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-13T22:06:27.883555748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:06:30.741594676Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-13T22:06:32.026982091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:06:32.864697303Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:06:32.864711475Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:06:32.884032054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:06:32.884049797Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:06:32.884059553Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:06:32.884105807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:06:33.956028723Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=debug ts=2026-05-13T22:06:34.312340388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:06:34.613367361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T22:06:36.228351827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:06:36.232864767Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:06:37.17054019Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-13T22:06:37.88361685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:06:39.046046128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:06:39.049420894Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:06:40.384420443Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-13T22:06:41.033331382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:06:41.036787144Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:06:42.094726981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:06:42.101731749Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:06:42.368461262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:06:42.372154991Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=warn ts=2026-05-13T22:06:42.883412132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:06:43.599550166Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-13T22:06:45.322893573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:06:45.326713926Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T22:06:45.533301852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:06:45.53697726Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:06:46.288272194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:06:46.292104591Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:06:46.813332481Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-13T22:06:47.042068002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:06:47.04591381Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:06:47.420213217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:06:47.423937123Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T22:06:47.88341158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:06:50.027781979Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-13T22:06:50.892150479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:06:50.897886257Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:06:52.883919005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:06:53.241829064Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-13T22:06:55.950258918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:06:55.954280683Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:06:55.954306434Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:06:55.954329569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:06:55.957374165Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:06:56.455885126Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-05-13T22:06:57.364490092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:06:57.371944479Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T22:06:57.883430665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:06:58.44628033Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:06:58.446327604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="50 kB" avg_comp="50 kB" total_uncomp="331 kB" avg_uncomp="331 kB" idle=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:06:58.450494586Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:06:58.878505096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:06:58.884520841Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T22:06:59.67061641Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-13T22:06:59.670641589Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=12
level=info ts=2026-05-13T22:07:02.883741021Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:07:02.883771771Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:07:02.883780383Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:07:02.883838788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:07:02.890748622Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-13T22:07:02.891961198Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007823791s
level=info ts=2026-05-13T22:07:03.463817383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:07:03.469877799Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:07:04.482926957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:07:04.486444249Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-13T22:07:04.616840285Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T22:07:07.029562366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:07:07.033118413Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:07:07.205691769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:07:07.211345777Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T22:07:07.883500567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:07:08.290847126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:46642"
level=info ts=2026-05-13T22:07:09.458149625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:07:09.462138207Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:07:11.654080446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="932 kB" avg_comp="932 kB" total_uncomp="7.4 MB" avg_uncomp="7.4 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-13T22:07:11.658918111Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e2318ada4%3A19e2340a6ed%3Aab503fa5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-13T22:07:11.658941162Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:07:11.658963675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:07:11.663036187Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:07:12.160513648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:07:12.168057288Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T22:07:12.883544494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T22:07:17.8839142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T22:07:22.883806255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:07:23.089341727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:07:23.095416207Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:07:23.333128588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=94aaa519e91a17a2 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:07:23.336629338Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/94aaa519e91a17a2/19e2322a134%3A19e23248595%3A1ff449f0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pixp\"}"
level=error ts=2026-05-13T22:07:23.336646288Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:07:23.336666439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:07:23.340292011Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T22:07:25.225670035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:07:25.227198181Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-13T22:07:27.884041953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:07:28.321687358Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:07:28.535321131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:07:28.539148909Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:07:28.539169882Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:07:28.539189726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:07:28.542193769Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:07:30.106064645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:07:30.109646934Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:07:30.329151087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:07:30.333309411Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:07:31.128661242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:07:31.132291864Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:07:32.864816047Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:07:32.865221216Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:07:32.883491667Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T22:07:32.883532182Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:07:32.88354949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:07:32.883557977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:07:32.883560792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:07:32.883627888Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-13T22:07:32.883635417Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T22:07:32.883654144Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.6µs success=true
level=debug ts=2026-05-13T22:07:32.883940089Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T22:07:34.619817221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T22:07:34.963801493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:07:34.971539483Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:07:34.971568365Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:07:34.971598471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:07:34.978470701Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:07:35.088851199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:07:35.092322807Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-13T22:07:35.929840641Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49242"
level=debug ts=2026-05-13T22:07:36.098766892Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T22:07:36.528806545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:07:36.532491629Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:07:37.694427223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:07:37.698185057Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T22:07:37.884203668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:07:38.104244707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:07:38.108139915Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:07:39.316195602Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-13T22:07:39.316217182Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=info ts=2026-05-13T22:07:41.821559733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:07:41.826174443Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:07:42.258500254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:07:42.262486291Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=debug ts=2026-05-13T22:07:42.527361639Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-13T22:07:42.883954369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:07:44.329779941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:07:44.333544253Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T22:07:45.253139815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:07:45.256816012Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:07:45.74268591Z caller=checkpoint.go:372 msg="writing series" size="573 kB"
level=info ts=2026-05-13T22:07:46.282796902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:07:46.286570446Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T22:07:47.884172046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:07:48.958635063Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-13T22:07:48.958658367Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=3
level=info ts=2026-05-13T22:07:49.726916955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:07:49.735155019Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:07:50.90614156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:07:50.910972825Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T22:07:52.170577354Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-13T22:07:52.884070934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:07:53.931744815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:07:53.945043333Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:07:53.945063812Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:07:53.9450871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:07:53.951478532Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:07:55.436991247Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-13T22:07:55.437043733Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=2
level=info ts=2026-05-13T22:07:56.509062828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:07:56.512697972Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:07:57.396197135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:07:57.402580226Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:07:57.860555029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:07:57.866458092Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T22:07:57.883621075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:07:58.35798296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:07:58.362087487Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:07:58.59891564Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-13T22:08:01.814849091Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T22:08:01.814878919Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T22:08:02.883931936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:08:02.883946195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:08:02.883958814Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:08:02.883968525Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T22:08:02.935257821Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:08:02.935316434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="176 kB" avg_comp="176 kB" total_uncomp="902 kB" avg_uncomp="902 kB" idle=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:08:02.939676562Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
ts=2026-05-13T22:08:04.623153202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T22:08:05.027059717Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=info ts=2026-05-13T22:08:06.552635174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:08:06.56103735Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T22:08:07.883993747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:08:08.241315354Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
ts=2026-05-13T22:08:08.294579387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55338"
level=info ts=2026-05-13T22:08:08.681134444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:08:08.689954617Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:08:09.75203636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:08:09.756493428Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T22:08:11.456070484Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T22:08:12.884160227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:08:13.457631662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:08:13.461478325Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:08:13.461500617Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:13.461521113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:08:13.464567725Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:08:14.08064645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:08:14.084824409Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=debug ts=2026-05-13T22:08:14.670522728Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-13T22:08:15.381127152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:08:15.384942029Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T22:08:17.883341275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:08:17.884639186Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T22:08:19.321805371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:08:19.32573372Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:08:19.325756007Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:19.325777414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:08:19.328851998Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:08:20.717792559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:08:20.724742443Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:08:21.099238556Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-13T22:08:22.297677834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:08:22.301260244Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:08:22.301281531Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:22.301302699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:08:22.304234998Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T22:08:22.883387059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:08:22.906585892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:08:22.910216438Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:08:22.910233232Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:22.910255373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:08:22.913239388Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T22:08:23.131088476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T22:08:24.312941618Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=debug ts=2026-05-13T22:08:27.526726794Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T22:08:27.8831933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:08:28.566826131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:08:28.570405956Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:08:28.570424107Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:28.570445444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:08:28.573321067Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:08:29.892640111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:08:29.89647086Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:08:29.896488723Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:29.896508473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:08:29.899454817Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:08:30.741758819Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T22:08:31.325378102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:08:31.328974952Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:08:31.785137714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:08:31.788700983Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:08:32.181908246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:08:32.185504349Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T22:08:32.185523116Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:32.18554245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:08:32.188495232Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:08:32.864404244Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:08:32.865456706Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T22:08:32.883790822Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:08:32.8838077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-13T22:08:32.883802319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:08:32.883816837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T22:08:32.998415861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:08:33.001844238Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:08:33.329621901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:08:33.332775921Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:08:33.524297439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:08:33.527811182Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=debug ts=2026-05-13T22:08:33.955497569Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-13T22:08:34.626428789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:08:36.124398475Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:08:36.555575858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:08:36.559735876Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T22:08:37.169823194Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T22:08:37.169844676Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=warn ts=2026-05-13T22:08:37.883707829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:08:40.385923581Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-13T22:08:40.385945169Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=info ts=2026-05-13T22:08:40.536589979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:08:40.543698441Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T22:08:42.883954958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:08:43.599057073Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-13T22:08:43.632064493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:08:43.636177304Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:08:43.636197279Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:43.636218102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:08:43.639423171Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:08:46.81362204Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-05-13T22:08:47.161648883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:08:47.165348123Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:08:47.72878976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:08:47.732943546Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T22:08:47.883902417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:08:49.572310989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:08:49.576818238Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:08:50.026879015Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-13T22:08:50.579449623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:08:50.5842996Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:08:50.638373395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:08:50.641835216Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:08:50.641854245Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:50.641874284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:08:50.645069099Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:08:51.154691939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:08:51.159081114Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:08:52.140851107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:08:52.153362882Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T22:08:52.884054169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:08:53.241393025Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-13T22:08:53.340345892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:08:53.346308947Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:08:53.346328072Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:53.346350061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:08:53.35165429Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:08:53.872892629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:08:53.876615387Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=info ts=2026-05-13T22:08:55.67372871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:08:55.677969579Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:08:56.456069015Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T22:08:56.761058229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:08:56.765096924Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:08:57.010739464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:08:57.017099812Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:08:57.017125664Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:08:57.017156128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:08:57.02280669Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T22:08:57.361865282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:08:57.365882675Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-13T22:08:57.883923645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:08:59.670692549Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T22:08:59.932363197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:08:59.936107168Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:09:00.591875839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="578 B" avg_comp="578 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:09:00.595517258Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:09:01.520438173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:09:01.524821968Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:09:02.883613394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:09:02.883615388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:09:02.883640874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:09:02.883648202Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T22:09:02.884839039Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-13T22:09:02.8848513Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-13T22:09:03.636570735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:09:03.641888194Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:09:03.641962186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:09:04.629614594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:09:04.674529205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:09:04.674583918Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T22:09:04.678216901Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:09:06.086260397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:09:06.091539295Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:09:06.098783991Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-13T22:09:06.847700218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:09:06.852770881Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T22:09:07.88418129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:09:09.209230107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:09:09.212926051Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:09:09.312809143Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T22:09:11.389526437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:09:11.393177657Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:09:11.430227357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:09:11.436802845Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T22:09:12.527264876Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=warn ts=2026-05-13T22:09:12.883913003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:09:13.085589669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:09:13.089188321Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:09:14.181458523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:09:14.185460785Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:09:15.741699941Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-13T22:09:16.09200163Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48300"
level=warn ts=2026-05-13T22:09:17.883701389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:09:18.955699188Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-13T22:09:22.170808644Z caller=checkpoint.go:372 msg="writing series" size="624 kB"
level=debug ts=2026-05-13T22:09:22.170832339Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-13T22:09:22.883874993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:09:23.221252964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:09:23.227434406Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:09:24.424877039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:09:24.429032855Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:09:25.176289733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:09:25.17973476Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:09:25.384456057Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=warn ts=2026-05-13T22:09:27.883672786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:09:28.598933647Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-13T22:09:29.083113667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:09:29.087279793Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:09:29.133493219Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:09:31.589605033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:09:31.594173269Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T22:09:31.816735886Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T22:09:31.816758149Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=info ts=2026-05-13T22:09:31.826428529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:09:31.830380342Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:09:31.830395897Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:09:31.830416044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:09:31.834112012Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:09:31.85833263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:09:31.863973215Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T22:09:32.864627625Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:09:32.864972079Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:09:32.883261699Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:09:32.883268307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:09:32.883281093Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:09:32.883290933Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-13T22:09:34.632779406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T22:09:35.027493556Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-13T22:09:35.936041737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60520"
level=info ts=2026-05-13T22:09:37.024383336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:09:37.028676876Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:09:37.750531949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:09:37.75419702Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T22:09:37.883895568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:09:38.241566895Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-13T22:09:39.190318938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:09:39.198294292Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:09:41.455983313Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-13T22:09:42.88356484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:09:43.773842006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:09:43.777967002Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=info ts=2026-05-13T22:09:44.398826839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:09:44.403430253Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:09:44.437505249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:09:44.443575787Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:09:44.671087248Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
level=info ts=2026-05-13T22:09:45.856147427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:09:45.865144911Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:09:46.017533914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:09:46.020866345Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T22:09:47.883633823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:09:47.88475779Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=debug ts=2026-05-13T22:09:47.884771144Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-13T22:09:50.433545196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:09:50.437634636Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:09:51.098698198Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-05-13T22:09:51.262234819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:09:51.265878369Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:09:51.812145948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:09:51.818805908Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:09:51.927288335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:09:51.930803465Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:09:52.583561896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:09:52.587847447Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T22:09:52.884105621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:09:53.232133562Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:09:54.312850042Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-05-13T22:09:55.957412767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:09:55.962635141Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:09:56.964597073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:09:56.968391788Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:09:57.526872284Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-13T22:09:57.579132216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:09:57.582287517Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T22:09:57.88361638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:09:59.910941982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:09:59.914770879Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:10:00.742168912Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-13T22:10:00.742188461Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-13T22:10:02.883128568Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:10:02.883152468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:10:02.883160423Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:10:02.883186272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:10:02.884796737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T22:10:03.956036557Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
ts=2026-05-13T22:10:04.635989388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:10:07.170077688Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-13T22:10:07.883540436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:10:08.301794873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60566"
level=info ts=2026-05-13T22:10:09.142429457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:10:09.147692713Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:10:10.384823866Z caller=checkpoint.go:372 msg="writing series" size="468 kB"
level=warn ts=2026-05-13T22:10:12.883355327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:10:13.422688447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:10:13.429516196Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T22:10:13.599343853Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-13T22:10:14.376480783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:10:14.380902808Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T22:10:16.096286797Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49810"
level=debug ts=2026-05-13T22:10:16.812865646Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=debug ts=2026-05-13T22:10:16.812886446Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-13T22:10:17.883579064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:10:18.948408862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:10:18.952571948Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=debug ts=2026-05-13T22:10:20.027102736Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-13T22:10:20.089556734Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:10:21.569589507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:10:21.574162645Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T22:10:22.883224796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:10:23.242296391Z caller=checkpoint.go:372 msg="writing series" size="688 kB"
level=info ts=2026-05-13T22:10:25.099124553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:10:25.104283161Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T22:10:26.456268113Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T22:10:26.456291338Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-13T22:10:26.664870442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:10:26.669596258Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=info ts=2026-05-13T22:10:27.355459517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:10:27.362174159Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T22:10:27.884154114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:10:27.938299292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:10:27.941626392Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:10:29.452688604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:10:29.457043571Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T22:10:29.669963611Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=info ts=2026-05-13T22:10:32.11667145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:10:32.123232678Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:10:32.864633067Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:10:32.864635475Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:10:32.884049093Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:10:32.884059757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:10:32.884080682Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:10:32.884090346Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T22:10:32.884132819Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=info ts=2026-05-13T22:10:33.191774808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:10:33.199617849Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:10:33.199640285Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:10:33.199679506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:10:33.206672843Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-13T22:10:34.638794991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:10:35.940400901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54160"
level=debug ts=2026-05-13T22:10:36.098595111Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T22:10:36.357285153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:10:36.360732033Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:10:37.226133156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:10:37.231708932Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T22:10:37.8838797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:10:38.305156957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57548"
level=info ts=2026-05-13T22:10:38.404518615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:10:38.408156654Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:10:39.312461023Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-13T22:10:39.312481835Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T22:10:40.209939244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:10:40.213858311Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:10:42.422947693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:10:42.426848641Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:10:42.527484254Z caller=checkpoint.go:372 msg="writing series" size="690 kB"
level=info ts=2026-05-13T22:10:42.63047809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:10:42.634532635Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-13T22:10:42.883878024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:10:45.741596777Z caller=checkpoint.go:372 msg="writing series" size="468 kB"
level=debug ts=2026-05-13T22:10:45.741622087Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-13T22:10:47.883571306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:10:48.514988632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="16 MB" avg_comp="2.3 MB" total_uncomp="139 MB" avg_uncomp="20 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:10:48.522876118Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:10:48.956124654Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-13T22:10:49.743002297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:10:49.747363793Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:10:50.312060193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:10:50.320041426Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:10:51.67712417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:10:51.680893244Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:10:52.170282535Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-13T22:10:52.883422156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:10:53.617221815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T22:10:53.618611789Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-13T22:10:53.621115497Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T22:10:53.747307069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:10:55.289130323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:10:55.292815238Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:10:55.384067874Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T22:10:57.883764244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:10:58.119221102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:10:58.126621455Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T22:10:58.615832859Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-13T22:10:58.615858213Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=4
level=info ts=2026-05-13T22:10:59.510295122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:10:59.514660978Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:11:01.812719146Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T22:11:02.883447724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:11:02.883476921Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:11:02.883484585Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:11:02.883475709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:11:04.641850022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:11:05.027594877Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
ts=2026-05-13T22:11:05.943675389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43030"
level=warn ts=2026-05-13T22:11:07.88393666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:11:08.241811985Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T22:11:10.788295075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:11:10.796538753Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=debug ts=2026-05-13T22:11:11.45632406Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-13T22:11:12.075275002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:11:12.083390746Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T22:11:12.883828682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:11:13.750796862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:11:13.75610648Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:11:14.431467231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:11:14.434812657Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:11:14.447872464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:11:14.451128714Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:11:14.530531091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:11:14.534929859Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:11:14.672190994Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T22:11:14.672213061Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
ts=2026-05-13T22:11:16.100066982Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:47774"
level=debug ts=2026-05-13T22:11:17.884239875Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T22:11:17.88425472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:11:18.284613659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:11:18.288497206Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:11:18.460035016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:11:18.4637508Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:11:21.100451153Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T22:11:21.100474494Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=warn ts=2026-05-13T22:11:22.883789895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:11:24.312869701Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-13T22:11:26.620662205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:11:26.624741649Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:11:27.527232147Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-13T22:11:27.768976617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:11:27.778510233Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-13T22:11:27.884245878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:11:28.884100208Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:11:30.742202701Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=info ts=2026-05-13T22:11:32.121267856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:11:32.125535334Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:11:32.86465049Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:11:32.864860625Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:11:32.883130992Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:11:32.883152254Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:11:32.883160562Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:11:32.883186527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:11:33.104379288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:11:33.108401445Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T22:11:33.956091418Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-05-13T22:11:34.645095703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:11:35.007764901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:11:35.014051252Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:11:35.790719211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:11:35.795168267Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T22:11:37.170417936Z caller=checkpoint.go:372 msg="writing series" size="354 kB"
level=warn ts=2026-05-13T22:11:37.884130227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:11:38.308874781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39346"
level=info ts=2026-05-13T22:11:40.07718968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:11:40.080996858Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:11:40.355870359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:11:40.359667854Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:11:40.38475807Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-13T22:11:40.56327767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:11:40.566805992Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T22:11:41.211482369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:11:41.217442375Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T22:11:42.883566326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:11:43.598685488Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-13T22:11:43.766213474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:11:43.770144734Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:11:46.203588549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:11:46.212368285Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:11:46.773010243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:11:46.774596753Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T22:11:46.812801566Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-13T22:11:46.839907723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:11:46.845613945Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-13T22:11:47.883383595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:11:50.02710486Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-13T22:11:52.884124641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:11:53.159738457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:11:53.164164663Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:11:53.241291967Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-13T22:11:53.585336517Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:11:56.4556485Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-13T22:11:56.45567166Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-13T22:11:56.581029468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:11:56.58496072Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:11:57.474632091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:11:57.481828497Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:11:57.70540362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:11:57.710273634Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T22:11:57.883317305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:11:57.975705052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:11:57.980978371Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:11:58.194132049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:11:58.198292926Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=debug ts=2026-05-13T22:11:59.670645985Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=info ts=2026-05-13T22:12:01.236532554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:12:01.240176018Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:12:02.366453882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:12:02.370198626Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:12:02.884043538Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:12:02.884041822Z caller=checkpoint.go:384 msg="flushing series" totalSize="657 kB" series=1
level=debug ts=2026-05-13T22:12:02.884067492Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:12:02.884075565Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:12:02.884102802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:12:02.890001045Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-13T22:12:02.891226678Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00744838s
ts=2026-05-13T22:12:04.648477819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=warn ts=2026-05-13T22:12:07.88343849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T22:12:12.883583408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:12:14.667948209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:12:14.673921639Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:12:17.737561051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:12:17.741855498Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:12:17.883763779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:12:20.256042559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:12:20.260170799Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:12:22.868861391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:12:22.874355305Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T22:12:22.883419268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:12:23.251247355Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:12:24.909782132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:12:25.553327312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:12:25.560845637Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:12:25.780409536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:12:25.783996224Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:12:25.825060826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:12:25.828567618Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:12:25.844619237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:12:25.848481961Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:12:27.833069339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:12:27.836533267Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T22:12:27.883470964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:12:28.434148828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:12:28.438077219Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:12:30.449069496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:12:30.455047202Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T22:12:32.864418605Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:12:32.865459965Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T22:12:32.883851233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:12:32.883869998Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:12:32.883865987Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T22:12:32.883877551Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:12:32.883870837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:12:32.88428306Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-13T22:12:32.884292649Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T22:12:32.884311979Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.166µs success=true
level=debug ts=2026-05-13T22:12:32.884689813Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T22:12:33.158602946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:12:33.162275049Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-13T22:12:34.651212928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:12:35.424073978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:12:35.431222196Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T22:12:35.950181099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34378"
level=info ts=2026-05-13T22:12:36.017671241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:12:36.021865258Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:12:36.099193409Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T22:12:37.88335194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:12:38.312967722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34396"
level=debug ts=2026-05-13T22:12:39.317197465Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-13T22:12:39.317220025Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=3
level=info ts=2026-05-13T22:12:40.039634826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:12:40.047196572Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:12:42.407629608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:12:42.411836958Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T22:12:42.528317763Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-13T22:12:42.883981426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:12:44.634796967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:12:44.639620101Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:12:44.96059912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:12:44.964071078Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:12:45.742486203Z caller=checkpoint.go:372 msg="writing series" size="655 kB"
level=info ts=2026-05-13T22:12:46.659773055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:12:46.663782179Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T22:12:47.883506335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:12:48.959099983Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-13T22:12:48.959121252Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=3
level=info ts=2026-05-13T22:12:49.651679663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:12:49.655793391Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:12:50.884417558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:12:50.888062235Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:12:52.171343959Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=info ts=2026-05-13T22:12:52.467112325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:12:52.4712287Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=warn ts=2026-05-13T22:12:52.88385391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:12:55.437423941Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-13T22:12:55.437453089Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=2
level=warn ts=2026-05-13T22:12:57.883742913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:12:58.599075404Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-13T22:12:59.173750473Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:12:59.409311951Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:13:01.102858264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:13:01.10676446Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:13:01.81441859Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T22:13:01.81445811Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T22:13:02.884156631Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:13:02.884183226Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:13:02.884192266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:13:02.884211511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:13:04.654215601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T22:13:05.027926256Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=info ts=2026-05-13T22:13:07.293630054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:13:07.29878379Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:13:07.883877969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:13:08.242858083Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
ts=2026-05-13T22:13:08.315434207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:49232"
level=info ts=2026-05-13T22:13:09.697428031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:13:09.701585516Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:13:10.144864215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:13:10.148536111Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:13:10.148557566Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:13:10.14859174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:13:10.151919779Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:13:10.272399694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:13:10.278539159Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:13:11.255450584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:13:11.263107699Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:13:11.456526105Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-13T22:13:11.505732553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:13:11.511726164Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-13T22:13:11.922901618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:13:11.92618707Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-13T22:13:12.883304285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:13:13.5651342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:13:13.571496216Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:13:14.37429043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:13:14.377512876Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:13:14.670456771Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-13T22:13:17.883764323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:13:17.884933598Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T22:13:20.162482345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:13:20.16636632Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:13:20.279761562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:13:20.283522116Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T22:13:21.099859047Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-13T22:13:21.579291791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:13:21.587321883Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-13T22:13:22.883540154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:13:24.313662172Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-05-13T22:13:27.527815273Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T22:13:27.883575733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:13:28.640964063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:13:28.645303219Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:13:30.673258325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:13:30.680309266Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:13:30.742400889Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T22:13:32.859737993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:13:32.863678011Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:13:32.864237988Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:13:32.865290226Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T22:13:32.883582308Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:13:32.883602581Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:13:32.883610774Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:13:32.883621881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:13:33.95678357Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-13T22:13:34.364106751Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-13T22:13:34.657230714Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:13:35.87537932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:13:35.87957925Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-13T22:13:36.097145823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:13:36.100830554Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:13:37.171255014Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T22:13:37.171276473Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=warn ts=2026-05-13T22:13:37.883561698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:13:38.118051727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:13:38.122596466Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:13:38.12261633Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:13:38.122638986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:13:38.126884854Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-13T22:13:38.318873561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51838"
level=debug ts=2026-05-13T22:13:40.387451788Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-13T22:13:40.387476632Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=info ts=2026-05-13T22:13:41.216642412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:13:41.220143422Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:13:41.220162236Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:13:41.220181021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:13:41.223128218Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:13:41.781749404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:13:41.785560004Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T22:13:42.519196145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-13T22:13:42.883869139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:13:43.598864193Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-13T22:13:43.782741381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:13:43.787807571Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
ts=2026-05-13T22:13:46.107978251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:40538"
level=debug ts=2026-05-13T22:13:46.813556578Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-13T22:13:47.43307047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:13:47.43698754Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:13:47.437007832Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:13:47.437051369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:13:47.440141309Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=warn ts=2026-05-13T22:13:47.883509869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:13:50.028077207Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-13T22:13:50.683043064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:13:50.688750925Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:13:51.00357222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:13:51.007059796Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T22:13:52.884062391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:13:53.241850642Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-13T22:13:54.527300843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:13:54.531214336Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:13:54.849984254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:13:54.855119279Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T22:13:56.456820181Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-13T22:13:57.883342153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:13:58.88587335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="578 B" avg_comp="578 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:13:58.889926297Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:13:59.467869279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:13:59.471312799Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:13:59.67154158Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T22:13:59.905648245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:13:59.910133842Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:13:59.910154884Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:13:59.910179224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:13:59.913702319Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:14:00.444045357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:14:00.4518638Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:14:00.502142651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:14:00.50560469Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:14:00.847347115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:14:00.850858724Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:14:01.232851938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:14:01.236193279Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:14:01.793842443Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:14:02.237976288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:14:02.241519502Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:14:02.88395662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:14:02.883982488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:14:02.883990656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:14:02.883986746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:14:02.885146993Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=debug ts=2026-05-13T22:14:02.885156102Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
ts=2026-05-13T22:14:04.660685589Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T22:14:06.099459013Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-13T22:14:07.88352688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:14:09.313348219Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T22:14:10.076430418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:14:10.085574886Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:14:10.085595148Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:14:10.085617428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:14:10.092423691Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:14:10.581735839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:14:10.588188298Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-13T22:14:12.527721825Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T22:14:12.852505321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:14:12.856311616Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T22:14:12.883907628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:14:14.342636319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:14:14.348423069Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:14:14.706406389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:14:14.709621131Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:14:15.742092595Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-13T22:14:16.111534081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:41244"
level=warn ts=2026-05-13T22:14:17.884139621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:14:18.7712955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:14:18.774696909Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-13T22:14:18.95653659Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-13T22:14:22.170993405Z caller=checkpoint.go:372 msg="writing series" size="590 kB"
level=debug ts=2026-05-13T22:14:22.171034856Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-13T22:14:22.883191269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:14:23.293070443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="17 MB" avg_uncomp="17 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:14:23.300504504Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:14:25.385304669Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=debug ts=2026-05-13T22:14:26.778259791Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:14:27.014984504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:14:27.019599249Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:14:27.019618695Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:14:27.019641204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:14:27.022956709Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=warn ts=2026-05-13T22:14:27.884141165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:14:28.59909614Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-13T22:14:29.533369666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:14:29.537904395Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:14:30.323227491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:14:30.331217515Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:14:30.331238237Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:14:30.33126587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:14:30.338080932Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:14:31.817392976Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T22:14:31.817416247Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=info ts=2026-05-13T22:14:31.96081547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:14:31.965612639Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:14:32.864614923Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:14:32.86532869Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:14:32.883659737Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:14:32.883680618Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:14:32.883688157Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:14:32.883741433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:14:33.111295623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:14:33.115296652Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:14:33.389394877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:14:33.393538605Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-13T22:14:34.663788449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:14:34.776519419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:14:34.781567084Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T22:14:35.02837363Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T22:14:35.327446229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:14:35.330872574Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:14:35.330894628Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:14:35.330915977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:14:35.333820373Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:14:36.017773994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:14:36.021428004Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:14:36.02144637Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:14:36.021465581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:14:36.024224724Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:14:36.23182544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:14:36.237899211Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:14:37.652687952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:14:37.656611246Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T22:14:37.8837033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:14:38.133154183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:14:38.137177915Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=debug ts=2026-05-13T22:14:38.242665922Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
ts=2026-05-13T22:14:38.322768846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43650"
level=info ts=2026-05-13T22:14:38.525339078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:14:38.528731991Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:14:41.456513825Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-13T22:14:42.884072635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:14:43.495496029Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:14:43.996693704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:14:44.004282417Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:14:44.671992602Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=info ts=2026-05-13T22:14:44.742773642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:14:44.747374714Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:14:44.875921381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:14:44.879460564Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:14:44.879482646Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:14:44.879503238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:14:44.882364678Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:14:46.627003073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:14:46.630653724Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T22:14:47.884092685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:14:47.885196989Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=debug ts=2026-05-13T22:14:51.099493627Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=info ts=2026-05-13T22:14:51.108546491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:14:51.112385353Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:14:51.398165295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:14:51.402063707Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:14:51.71184816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:14:51.715427461Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:14:51.715448222Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:14:51.715470189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:14:51.71853937Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:14:52.884091483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:14:53.355556569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:14:53.362563577Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:14:54.313908539Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-13T22:14:54.313932199Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-13T22:14:55.660802231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:14:55.668335547Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:14:57.528286135Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T22:14:57.883970683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:14:59.208008191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:14:59.212181572Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:15:00.086068076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:15:00.089493765Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:15:00.089515344Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:15:00.089536469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:15:00.092816342Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:15:00.743336324Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T22:15:00.743358101Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-13T22:15:02.073755589Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:15:02.222140146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:15:02.225779067Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:15:02.225800836Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:15:02.225822688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:15:02.228807393Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:15:02.612843896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:15:02.618426543Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e2318497d%3A19e2349691d%3Abc1cc97?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-13T22:15:02.618448436Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:15:02.618470598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:15:02.622097976Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-13T22:15:02.88359679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:15:02.883605477Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:15:02.883630714Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:15:02.883640379Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T22:15:03.956523411Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
ts=2026-05-13T22:15:04.667105863Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T22:15:05.803649443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-13T22:15:05.807345441Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e23184b19%3A19e2318ada5%3Aa529d62c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-13T22:15:05.807371615Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:15:05.807395061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:15:05.810973937Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T22:15:05.967679658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:15:05.971153934Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T22:15:07.171037752Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-13T22:15:07.883476056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:15:10.385346821Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
level=info ts=2026-05-13T22:15:11.357000352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:15:11.361234552Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=warn ts=2026-05-13T22:15:12.884133872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:15:13.378371094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:15:13.383169582Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T22:15:13.599973413Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-13T22:15:15.545945088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:15:15.551388326Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:15:16.813790373Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-13T22:15:16.81381207Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-13T22:15:17.312220311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:15:17.316742721Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T22:15:17.883694184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:15:18.767067792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:15:18.77190026Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-13T22:15:19.749408055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:15:19.753871496Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T22:15:20.02771241Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T22:15:20.582285565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:15:20.58584618Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:15:20.594890383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:15:20.598834085Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:15:21.05213192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:15:21.055778019Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:15:21.63318211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:15:21.640035483Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T22:15:22.883632306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:15:23.242650965Z caller=checkpoint.go:372 msg="writing series" size="689 kB"
level=debug ts=2026-05-13T22:15:24.091177263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:15:24.092908345Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T22:15:25.108969948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:15:25.113204604Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:15:25.647502684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:15:25.651345593Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:15:26.457040305Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T22:15:26.457063658Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-13T22:15:26.90918902Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:15:27.883199639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:15:28.422514338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:15:28.426360746Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:15:28.426379892Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:15:28.426402817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:15:28.429295588Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:15:29.265259449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:15:29.269200928Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T22:15:29.670824566Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=info ts=2026-05-13T22:15:29.818564195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:15:29.824377541Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:15:29.824396441Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:15:29.824417843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:15:29.829807377Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:15:30.772958681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:15:30.776819455Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:15:32.458274198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:15:32.46241391Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:15:32.805230922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:15:32.808840787Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:15:32.808863106Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:15:32.808883572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:15:32.811800429Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:15:32.864672297Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:15:32.864868944Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=debug ts=2026-05-13T22:15:32.867213639Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-13T22:15:32.867254491Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976344 n=1
level=debug ts=2026-05-13T22:15:32.867296737Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976344 n=1
level=info ts=2026-05-13T22:15:32.883128999Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:15:32.883146325Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:15:32.883153926Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:15:32.883183188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:15:32.885296158Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=info ts=2026-05-13T22:15:33.797131004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:15:33.800819448Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:15:33.800838719Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:15:33.800859938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:15:33.803950682Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:15:33.877378252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:15:33.880785346Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T22:15:34.669899365Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
ts=2026-05-13T22:15:35.959646935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33570"
level=debug ts=2026-05-13T22:15:36.099717075Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-13T22:15:37.884140047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:15:39.313777527Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-13T22:15:39.313798904Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T22:15:39.571380459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:15:39.575295631Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:15:42.314666427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:15:42.321460755Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:15:42.528617114Z caller=checkpoint.go:372 msg="writing series" size="700 kB"
level=warn ts=2026-05-13T22:15:42.883779975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:15:44.181822815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:15:44.186080005Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:15:44.230132574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:15:44.237781846Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:15:44.735435344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:15:44.739699505Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-13T22:15:45.134914808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:15:45.13834528Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:15:45.742258913Z caller=checkpoint.go:372 msg="writing series" size="470 kB"
level=debug ts=2026-05-13T22:15:45.742284485Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
ts=2026-05-13T22:15:46.117105085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:36762"
level=info ts=2026-05-13T22:15:46.46631213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:15:46.470778071Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:15:47.305082095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:15:47.309074348Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-13T22:15:47.883614522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:15:48.266595245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:15:48.270098796Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:15:48.409609975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:15:48.41523308Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:15:48.956637019Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-13T22:15:52.04934758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:15:52.053424444Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=debug ts=2026-05-13T22:15:52.170920442Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-13T22:15:52.883481264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:15:53.075455395Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:15:55.384831478Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-13T22:15:57.883219298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:15:58.617916542Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-13T22:15:58.617940866Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=4
level=debug ts=2026-05-13T22:16:01.813848137Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T22:16:02.467059322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:16:02.472466375Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:16:02.472486306Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:16:02.472509194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:16:02.475978311Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:16:02.883703393Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:16:02.883730746Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:16:02.883738789Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:16:02.88374329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:16:02.911130036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-13T22:16:04.672681319Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T22:16:05.027503764Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
ts=2026-05-13T22:16:05.963180229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47520"
level=warn ts=2026-05-13T22:16:07.883780426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:16:08.242387484Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T22:16:09.744139203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:16:09.750256904Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:16:10.18862865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:16:10.192777032Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:16:11.456425137Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=warn ts=2026-05-13T22:16:12.883275783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:16:14.215612504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:16:14.221407193Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:16:14.245444843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:16:14.249481345Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:16:14.671968876Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T22:16:14.671992793Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=info ts=2026-05-13T22:16:15.11575218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:16:15.122130214Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:16:15.48006541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:16:15.483739811Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T22:16:16.120258185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:35912"
level=info ts=2026-05-13T22:16:16.773295715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:16:16.782346766Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:16:17.26590924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:16:17.269856946Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:16:17.464427747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:16:17.468215274Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:16:17.883904639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:16:17.884991527Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-13T22:16:18.018099071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:16:18.022732685Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:16:19.811251949Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:16:20.398063159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:16:20.40196292Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:16:21.100575406Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T22:16:21.100598707Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=info ts=2026-05-13T22:16:21.569403817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:16:21.573085881Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:16:22.620908599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:16:22.625595531Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T22:16:22.883913895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:16:23.924703411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:16:23.928351616Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:16:24.000483402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:16:24.003948953Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:16:24.31384843Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-13T22:16:24.372128917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:16:24.375394716Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:16:27.528180087Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-13T22:16:27.883701418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:16:30.742856573Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
level=info ts=2026-05-13T22:16:32.306130264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:16:32.31076995Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=debug ts=2026-05-13T22:16:32.864419884Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:16:32.865460443Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T22:16:32.883838647Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:16:32.883842999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:16:32.883854726Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:16:32.883861797Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T22:16:33.956860758Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-05-13T22:16:34.676176698Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T22:16:34.998388838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:16:35.006448041Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:16:36.77059331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:16:36.774523711Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T22:16:37.042291503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:16:37.053128536Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:16:37.170706422Z caller=checkpoint.go:372 msg="writing series" size="418 kB"
level=info ts=2026-05-13T22:16:37.305958327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:16:37.3100578Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-13T22:16:37.883720421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:16:38.248962319Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T22:16:40.385443254Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-13T22:16:41.199930526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:16:41.203973977Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=warn ts=2026-05-13T22:16:42.883457226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:16:43.599840603Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-13T22:16:46.183674205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:16:46.188711163Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:16:46.81333126Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=warn ts=2026-05-13T22:16:47.883813039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:16:50.02796277Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=debug ts=2026-05-13T22:16:50.027988107Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-13T22:16:50.094270419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:16:50.098860893Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T22:16:52.883870822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:16:53.241728408Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-13T22:16:55.492387884Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:16:55.502387842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:16:55.50833799Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:16:56.456325642Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-13T22:16:57.883311984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:16:58.708123854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:16:58.711961696Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:16:58.809124196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:16:58.812989662Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:16:59.671178304Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=info ts=2026-05-13T22:17:02.883971927Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:17:02.883999118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:17:02.884006894Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:17:02.884030812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:17:02.885038825Z caller=checkpoint.go:384 msg="flushing series" totalSize="804 kB" series=3
level=info ts=2026-05-13T22:17:02.891786379Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-13T22:17:02.893247795Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00880292s
ts=2026-05-13T22:17:04.678918678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T22:17:04.880005051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:17:04.883991066Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:17:05.125586827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:17:05.129328707Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:17:05.763088032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:17:05.766696991Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:17:06.0657859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:17:06.069267994Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:17:07.839883582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:17:07.843460181Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T22:17:07.883945329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:17:08.215078198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:17:08.219875793Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:17:11.619399815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:17:11.623312679Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T22:17:12.883558023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:17:13.585435317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:17:13.589201302Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:17:14.647028088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:17:14.654877293Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:17:15.096425849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:17:15.100090937Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-13T22:17:16.124613397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:55996"
level=info ts=2026-05-13T22:17:16.246445711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:17:16.251678708Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:17:17.62664496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:17:17.63203325Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=warn ts=2026-05-13T22:17:17.88341397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:17:19.994180237Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:17:22.883736018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:17:24.760293131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:17:24.767482861Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:17:24.994300528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:17:24.998467813Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-13T22:17:27.88347563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:17:28.664056516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:17:29.621814339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:17:29.626173304Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T22:17:30.568062112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:17:30.572165291Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=info ts=2026-05-13T22:17:30.651263162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:17:30.655192246Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:17:32.810475025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:17:32.817852373Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:17:32.817875896Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:17:32.817918207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:17:32.824842542Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:17:32.864722849Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:17:32.865097498Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:17:32.883355777Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:17:32.883371877Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:17:32.883369565Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T22:17:32.883379758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:17:32.883467775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:17:32.883523972Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-13T22:17:32.88353473Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T22:17:32.883551774Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.385µs success=true
level=debug ts=2026-05-13T22:17:32.883889426Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
ts=2026-05-13T22:17:34.682104455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:17:36.098455537Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-13T22:17:37.883823559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:17:39.314712814Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-13T22:17:39.314733843Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=debug ts=2026-05-13T22:17:42.527329085Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-05-13T22:17:42.746836534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:17:42.750712458Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T22:17:42.883799037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:17:43.41569644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:17:43.421542169Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:17:45.741613208Z caller=checkpoint.go:372 msg="writing series" size="722 kB"
level=info ts=2026-05-13T22:17:46.355274081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:17:46.360442121Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T22:17:47.884049926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:17:48.226394337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:17:48.230255799Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:17:48.95684501Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-13T22:17:48.956865496Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=3
level=info ts=2026-05-13T22:17:49.359291598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:17:49.362804256Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:17:50.545790633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:17:50.549310208Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:17:50.569501349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:17:50.576820947Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:17:51.212628711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:17:51.216336565Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:17:52.170029169Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=warn ts=2026-05-13T22:17:52.883375392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:17:53.705760702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:17:53.709586625Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:17:54.144932475Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:17:55.439610147Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-13T22:17:55.439635962Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=2
level=info ts=2026-05-13T22:17:57.437482011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:17:57.442567379Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T22:17:57.884309948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:17:58.598227062Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-13T22:17:58.891149846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:17:58.895222924Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:18:00.348852927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:18:00.352645914Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:18:01.661253586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:18:01.664852706Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:18:01.814609623Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T22:18:01.814633388Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T22:18:02.883975146Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:18:02.884003512Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:18:02.884023061Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:18:02.884009746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:18:03.426467298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:18:03.435763706Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-13T22:18:04.685118473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:18:05.027560154Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-13T22:18:07.096958097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:18:07.101035684Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:18:07.883481167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:18:08.241137884Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=info ts=2026-05-13T22:18:09.972065987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:18:09.975753803Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:18:11.455804062Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-13T22:18:12.365609231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:18:12.370161938Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T22:18:12.883264193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:18:14.670098387Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-13T22:18:14.877888276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:18:14.885835277Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:18:17.67964483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:18:17.685628511Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-13T22:18:17.883753852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:18:17.883930026Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-13T22:18:18.928359736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:18:20.133742553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:18:20.143538752Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:18:20.149596552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:18:20.153541407Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-13T22:18:21.098370588Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-13T22:18:22.883498282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:18:23.400420758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:18:23.404701562Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:18:23.571212291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:18:23.576967308Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:18:24.312844294Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=debug ts=2026-05-13T22:18:25.839817804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-13T22:18:27.526809689Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T22:18:27.88336033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:18:28.137394685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:18:28.14129674Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=debug ts=2026-05-13T22:18:30.741545523Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T22:18:31.263280116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:18:31.266982766Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:18:32.864719673Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:18:32.865085825Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:18:32.883335147Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:18:32.883350342Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:18:32.883357758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:18:32.883372087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:18:33.956119667Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-13T22:18:34.687932963Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T22:18:37.169875861Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T22:18:37.169903781Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-13T22:18:37.884068695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:18:39.583869917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:18:39.587869736Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:18:40.384418197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:18:40.387137902Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-13T22:18:40.387158965Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=error ts=2026-05-13T22:18:40.388303486Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:18:40.448405342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:18:40.452261658Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:18:42.296439563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:18:42.300381201Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-13T22:18:42.883611731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:18:42.88666703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:18:42.890061038Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:18:43.598131696Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-13T22:18:44.768617193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:18:44.772609042Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:18:45.383340703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:18:45.38779043Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T22:18:46.813328085Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-13T22:18:46.928721028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:18:46.937200858Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:18:47.06081068Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:18:47.884209532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:18:50.02678408Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-13T22:18:50.12215185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:18:50.126511952Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:18:51.943069083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:18:51.947522106Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:18:52.554168269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:18:52.557904222Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-05-13T22:18:52.883438368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:18:53.241907673Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=debug ts=2026-05-13T22:18:56.455240567Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T22:18:57.022889273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:18:57.026691375Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-13T22:18:57.883671783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:18:58.156059214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:18:58.160430502Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:18:59.671157924Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T22:19:01.125675202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:19:01.134191871Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:19:01.539615113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:19:01.54121152Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-13T22:19:02.309683641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:19:02.31414487Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-13T22:19:02.884028461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:19:02.884058145Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:19:02.884066233Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:19:02.884111683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:19:02.884195187Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-13T22:19:02.884208956Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
ts=2026-05-13T22:19:04.693152892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:19:05.42352141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:19:05.431543067Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:19:05.618065164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="578 B" avg_comp="578 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:19:05.621607122Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:19:06.098163202Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-13T22:19:07.107124124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:19:07.11140218Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-13T22:19:07.883858573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:19:09.313462995Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-13T22:19:12.527709842Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=warn ts=2026-05-13T22:19:12.883303901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:19:13.211697488Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:19:15.741574921Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
ts=2026-05-13T22:19:16.13266685Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:49180"
level=info ts=2026-05-13T22:19:17.346546491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:19:17.354398596Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=warn ts=2026-05-13T22:19:17.883319956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:19:18.955602603Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T22:19:19.259910246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:19:19.264544621Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:19:20.32032217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:19:20.328479393Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:19:20.576162353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:19:20.580046534Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:19:22.170138534Z caller=checkpoint.go:372 msg="writing series" size="503 kB"
level=debug ts=2026-05-13T22:19:22.170160229Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-13T22:19:22.883484426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:19:23.289998682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:19:23.29442046Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:19:25.141009754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:19:25.145600208Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T22:19:25.384237895Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-13T22:19:26.432143743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:19:26.43666906Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:19:26.843673071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:19:26.847104065Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T22:19:27.883755918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:19:28.598598891Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-13T22:19:29.370992793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:19:29.375315386Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:19:31.815369627Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T22:19:31.815391546Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=info ts=2026-05-13T22:19:32.864662815Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:19:32.865340826Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:19:32.883715062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:19:32.88371388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:19:32.883730988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:19:32.883737908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T22:19:32.938026779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:19:32.9414481Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:19:32.941463777Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:19:32.941481961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:19:32.944338189Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-13T22:19:34.695842691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T22:19:35.027563181Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-13T22:19:36.595082155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:19:36.598979065Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:19:37.883651048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:19:38.241660772Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=info ts=2026-05-13T22:19:38.360969052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:19:38.364936458Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:19:39.796677562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:19:39.804408342Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:19:40.610025483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:19:40.61381656Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T22:19:41.456049216Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-13T22:19:41.75483281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:19:41.758436458Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=warn ts=2026-05-13T22:19:42.883285276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:19:44.451456538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:19:44.454893354Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:19:44.669940168Z caller=checkpoint.go:372 msg="writing series" size="494 kB"
level=info ts=2026-05-13T22:19:45.485826601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:19:45.489295581Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-13T22:19:46.135733847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:57576"
level=debug ts=2026-05-13T22:19:47.126291772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:19:47.33400234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:19:47.337472447Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:19:47.412814694Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:19:47.88357303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:19:47.88470934Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=debug ts=2026-05-13T22:19:51.098716839Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-13T22:19:52.883200504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:19:53.281483031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:19:53.290846471Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:19:54.312650269Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-13T22:19:54.312672642Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-13T22:19:55.399790491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:19:55.406975262Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:19:55.481276353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:19:55.485445824Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-13T22:19:55.821169083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:19:55.828598536Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:19:57.527614963Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T22:19:57.883384692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:20:00.742128072Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T22:20:00.742150738Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-13T22:20:00.892670767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:20:00.897210742Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:20:02.278045952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:20:02.284630326Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:20:02.883851543Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:20:02.883880915Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:20:02.88388918Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:20:02.883925347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:20:03.956067084Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
level=info ts=2026-05-13T22:20:04.069595733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:20:04.074445866Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-13T22:20:04.69829114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T22:20:05.572972891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:20:05.578069859Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:20:06.268251596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:20:06.272441394Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=info ts=2026-05-13T22:20:06.996459432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:20:07.001377737Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:20:07.001397157Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:20:07.001419122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:20:07.005480918Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T22:20:07.170141061Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-13T22:20:07.883545908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:20:08.885846051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:20:08.889639292Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:20:08.889658163Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:20:08.889676783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:20:08.892794852Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:20:10.198356136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:20:10.202349624Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:20:10.202370109Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:20:10.202390776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:20:10.205675957Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T22:20:10.38437092Z caller=checkpoint.go:372 msg="writing series" size="476 kB"
level=info ts=2026-05-13T22:20:10.719169024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:20:10.722477933Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-13T22:20:12.883363614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:20:13.598483076Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-13T22:20:14.181469432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:20:14.185609954Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:20:16.813229738Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-13T22:20:16.813254604Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-13T22:20:16.888563379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:20:16.892180632Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T22:20:17.338375339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:20:17.883394272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:20:20.027467984Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T22:20:20.382561417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:20:20.386972019Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:20:21.563130083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:20:21.568175562Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T22:20:22.883648519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:20:23.105872906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:20:23.109689692Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:20:23.242269246Z caller=checkpoint.go:372 msg="writing series" size="689 kB"
level=info ts=2026-05-13T22:20:23.656679368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:20:23.661769717Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-13T22:20:24.131063099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:20:24.134430243Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:20:26.121559671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:20:26.125053505Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:20:26.456113255Z caller=checkpoint.go:372 msg="writing series" size="512 kB"
level=debug ts=2026-05-13T22:20:26.456136487Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-13T22:20:27.564046341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:20:27.570746305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T22:20:27.884007032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:20:29.670540807Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=info ts=2026-05-13T22:20:32.031817779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:20:32.035604013Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:20:32.035624509Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:20:32.035646553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:20:32.038784571Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:20:32.864627558Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:20:32.864892303Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-13T22:20:32.883171555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:20:32.883182254Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:20:32.883201352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:20:32.883212156Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T22:20:32.884307972Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
ts=2026-05-13T22:20:34.701855622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
ts=2026-05-13T22:20:35.977720194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34492"
level=debug ts=2026-05-13T22:20:36.098788158Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T22:20:37.195568585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:20:37.200189218Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-13T22:20:37.883514455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:20:37.944627296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:20:37.948279109Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:20:39.312643743Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-13T22:20:39.312668297Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-13T22:20:40.238396949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:20:40.245602861Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:20:42.306562437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:20:42.310824884Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:20:42.527716115Z caller=checkpoint.go:372 msg="writing series" size="705 kB"
level=info ts=2026-05-13T22:20:42.641615124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:20:42.645531301Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=warn ts=2026-05-13T22:20:42.884147123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:20:44.710403668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:20:44.714714991Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T22:20:45.465026254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:20:45.469411635Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:20:45.741153355Z caller=checkpoint.go:372 msg="writing series" size="434 kB"
level=debug ts=2026-05-13T22:20:45.741178152Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-13T22:20:46.549167676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:20:46.555131934Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T22:20:47.883458477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:20:48.955584231Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-13T22:20:49.478781911Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:20:50.324005858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:20:50.331434047Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:20:50.331454911Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:20:50.331474989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:20:50.337608655Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:20:52.169863736Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-13T22:20:52.70670433Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:20:52.706750882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T22:20:52.71048904Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e2318477d%3A19e236b1635%3Ae109ddfe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-13T22:20:52.883741278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:20:55.175053343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:20:55.181099812Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:20:55.248126988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:20:55.251813487Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:20:55.384357146Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-13T22:20:57.281830996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:20:57.285943888Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:20:57.285968528Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:20:57.285995706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:20:57.289197996Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T22:20:57.884215392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:20:58.621929512Z caller=checkpoint.go:372 msg="writing series" size="20 MB"
level=debug ts=2026-05-13T22:20:58.621954233Z caller=checkpoint.go:384 msg="flushing series" totalSize="20 MB" series=4
level=info ts=2026-05-13T22:20:58.875913628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:20:58.880656967Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T22:21:01.813187082Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-13T22:21:02.148328553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:21:02.156928501Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:21:02.598004352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:21:02.602099527Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:21:02.883739403Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:21:02.883771444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:21:02.883781807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:21:02.883788341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:21:03.76950287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:21:03.773268414Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:21:04.374306973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d727ae70a9db41a7 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="902 kB" avg_uncomp="902 kB" full=1 labels="{stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:21:04.378597259Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d727ae70a9db41a7/19e23235cb3%3A19e232ad115%3Aa3b5d2fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-nstw\"}"
level=error ts=2026-05-13T22:21:04.378618161Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:04.378641363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9173a55ce9d79e07 immediate=false num_chunks=1 total_comp="829 kB" avg_comp="829 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" idle=1 labels="{stream=\"build-e2e-rvxv\"}"
level=error ts=2026-05-13T22:21:04.393520862Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9173a55ce9d79e07/19e2322d3fd%3A19e2333e7c8%3A80eb53d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rvxv\"}"
ts=2026-05-13T22:21:04.704615459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=debug ts=2026-05-13T22:21:05.027265407Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-13T22:21:07.400854489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:21:07.404507352Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:21:07.404526123Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:07.404547399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:21:07.407585886Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-13T22:21:07.883342087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:21:07.918561569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:21:07.922421126Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:21:07.984586886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T22:21:07.988790108Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e2318477d%3A19e236b1635%3Ae109ddfe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T22:21:08.240996595Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-13T22:21:09.706448766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:21:09.710338289Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:21:11.455258876Z caller=checkpoint.go:372 msg="writing series" size="370 B"
level=info ts=2026-05-13T22:21:11.81031026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:21:11.817379399Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:21:11.817402177Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:11.81742828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:21:11.824400252Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T22:21:12.883212451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:21:13.385380999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:21:13.389234192Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:21:14.329395026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:21:14.333066151Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-13T22:21:14.671076725Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-13T22:21:14.67109873Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=debug ts=2026-05-13T22:21:17.883811134Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-13T22:21:17.883824377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:21:19.676963724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9173a55ce9d79e07 immediate=false num_chunks=1 total_comp="837 kB" avg_comp="837 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"build-e2e-rvxv\"}"
level=error ts=2026-05-13T22:21:19.681919383Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9173a55ce9d79e07/19e2322d3fd%3A19e2333e7c8%3A80eb53d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rvxv\"}"
level=info ts=2026-05-13T22:21:19.82113801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:21:19.824801307Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:21:19.824827662Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:19.824852085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:21:19.827916177Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:21:20.836661245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:21:20.840186674Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:21:20.840204174Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:20.840222433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:21:20.843070217Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T22:21:21.101414848Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-13T22:21:21.10144055Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=info ts=2026-05-13T22:21:21.86421368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:21:21.872723768Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:21:22.725799098Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:21:22.88320272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:21:24.312553344Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=debug ts=2026-05-13T22:21:24.552173812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:21:25.226582953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:21:25.238574586Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:21:25.61064601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:21:25.614093236Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:21:26.457606113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:21:26.460823339Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-13T22:21:27.527699236Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-13T22:21:27.883194672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:21:28.393418775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:21:28.396674627Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:21:28.396692642Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:28.396711622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:21:28.399418266Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:21:29.825603107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:21:29.833342795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:21:30.74130169Z caller=checkpoint.go:372 msg="writing series" size="363 kB"
level=info ts=2026-05-13T22:21:30.917813552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:21:30.921640706Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:21:31.610405702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:21:31.614791211Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:21:31.614812844Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:31.614837155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:21:31.618607444Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-13T22:21:32.864765946Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:21:32.865085761Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:21:32.883355108Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:21:32.883371922Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:21:32.883380774Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:21:32.883390529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:21:33.956163735Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
ts=2026-05-13T22:21:34.707914072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T22:21:34.882054333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:21:34.885734818Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:21:34.885751731Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:34.88577018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:21:34.888596904Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
ts=2026-05-13T22:21:35.981882224Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55266"
level=debug ts=2026-05-13T22:21:37.169828325Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=warn ts=2026-05-13T22:21:37.884081532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:21:38.59907003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:21:38.603721907Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:21:38.603743404Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:38.603763733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:21:38.607659971Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T22:21:38.767180909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:21:38.770841948Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:21:38.872284717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T22:21:38.876846632Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e2318477d%3A19e236b1635%3Ae109ddfe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-13T22:21:40.208276889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:21:40.211938477Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:21:40.384499637Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-13T22:21:42.776219267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:21:42.779926473Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-13T22:21:42.883894683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:21:43.598809879Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-13T22:21:43.936235747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:21:43.940076869Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:21:44.393892921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:21:44.400749509Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:21:44.400767587Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:44.400787106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:21:44.406874818Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:21:45.166063148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:21:45.169624226Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:21:46.420917588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:21:46.42518745Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-13T22:21:46.813333978Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=warn ts=2026-05-13T22:21:47.883868211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:21:48.128550155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:21:48.132774813Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-13T22:21:49.241967227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:21:49.245390379Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:21:50.026964978Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=debug ts=2026-05-13T22:21:50.48125257Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:21:51.477719669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:21:51.482108671Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:21:51.884344999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:21:51.888062245Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:21:52.184846236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:21:52.189538278Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T22:21:52.883996351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:21:53.241546753Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-13T22:21:56.455607077Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-13T22:21:56.455627573Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-13T22:21:56.750370418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:21:56.757179316Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-13T22:21:56.903737349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9173a55ce9d79e07 immediate=false num_chunks=1 total_comp="837 kB" avg_comp="837 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"build-e2e-rvxv\"}"
level=error ts=2026-05-13T22:21:56.908953355Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9173a55ce9d79e07/19e2322d3fd%3A19e2333e7c8%3A80eb53d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rvxv\"}"
level=info ts=2026-05-13T22:21:57.573643228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:21:57.58210403Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T22:21:57.883621486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:21:57.9578421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:21:57.961626857Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e2318a1e3%3A19e2318ecd5%3A46941a1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-13T22:21:57.961648092Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:21:57.961668701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="22 kB" avg_uncomp="22 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-13T22:21:57.964676279Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e2318a1e3%3A19e233176c7%3Ae56467ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-13T22:21:58.033792557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:21:58.038034187Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T22:21:59.670286661Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=info ts=2026-05-13T22:22:02.001420052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:22:02.00810377Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:22:02.716057274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:22:02.719525862Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:22:02.883713413Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:22:02.883722145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:22:02.883746078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-13T22:22:02.883747156Z caller=checkpoint.go:384 msg="flushing series" totalSize="657 kB" series=1
level=info ts=2026-05-13T22:22:02.883754767Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T22:22:02.890058153Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-13T22:22:02.891322128Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007610133s
level=info ts=2026-05-13T22:22:03.44151638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:22:03.444838807Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:22:03.444857885Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:22:03.444878517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:22:03.44763123Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-13T22:22:04.710923664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=info ts=2026-05-13T22:22:04.819608721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:22:04.823161799Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:22:07.227258856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:22:07.231142896Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-13T22:22:07.883509553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-13T22:22:08.343734957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60674"
level=info ts=2026-05-13T22:22:09.554140137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:22:09.561850762Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-13T22:22:09.850490366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:22:09.857707678Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:22:10.335510495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:22:11.581129039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-13T22:22:11.58505056Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e2318a1e3%3A19e233176c7%3Ae56467ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-05-13T22:22:12.884150511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:22:13.807290298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:22:13.811264946Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-05-13T22:22:17.450138992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:22:17.454476847Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-13T22:22:17.884182673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-13T22:22:22.883966228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:22:22.998397467Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:22:23.243443198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:22:23.247901608Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:22:23.247923392Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:22:23.247943433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:22:23.251475795Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:22:23.431931754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:22:23.435796825Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:22:23.663660035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:22:23.667889442Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T22:22:26.878205445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:22:26.884231467Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-13T22:22:27.883370526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:22:30.436909391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T22:22:30.441355711Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e2318477d%3A19e236b1635%3Ae109ddfe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T22:22:32.864737171Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:22:32.864735304Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T22:22:32.883077448Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-13T22:22:32.883208391Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-13T22:22:32.883216442Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-13T22:22:32.883235255Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.234µs success=true
level=info ts=2026-05-13T22:22:32.883309856Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:22:32.883309921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:22:32.883326758Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:22:32.883333897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-13T22:22:32.883536284Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-13T22:22:33.027637102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:22:33.032704589Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:22:33.032725133Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:22:33.032747321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:22:33.036450392Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T22:22:34.713816562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-c0eeadc2 10.130.0.69:7946"
level=info ts=2026-05-13T22:22:35.765342919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:22:35.769291166Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:22:35.903790651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-13T22:22:35.907671162Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e2318a1e3%3A19e233176c7%3Ae56467ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-05-13T22:22:35.986320388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51268"
level=debug ts=2026-05-13T22:22:36.098240499Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-13T22:22:36.29276064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:22:36.296558376Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-13T22:22:36.420794223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:22:36.426546798Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:22:37.073470829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:22:37.077339527Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-13T22:22:37.883834711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:22:39.316389954Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-13T22:22:39.316412937Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=3
level=info ts=2026-05-13T22:22:39.357198455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:22:39.360740667Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:22:42.526750875Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=warn ts=2026-05-13T22:22:42.883217941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:22:43.91232887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:22:43.916384868Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:22:44.578382276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:22:44.581599303Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:22:44.740044677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:22:44.743473424Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=debug ts=2026-05-13T22:22:45.741367216Z caller=checkpoint.go:372 msg="writing series" size="586 kB"
ts=2026-05-13T22:22:46.146134413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:48976"
level=warn ts=2026-05-13T22:22:47.883627817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:22:47.911782341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:22:47.916863843Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:22:48.188994163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9173a55ce9d79e07 immediate=false num_chunks=1 total_comp="837 kB" avg_comp="837 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"build-e2e-rvxv\"}"
level=error ts=2026-05-13T22:22:48.19443237Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9173a55ce9d79e07/19e2322d3fd%3A19e2333e7c8%3A80eb53d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rvxv\"}"
level=debug ts=2026-05-13T22:22:48.956686085Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-13T22:22:48.956709607Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=3
level=info ts=2026-05-13T22:22:51.059141919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:22:51.062793275Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:22:51.677852499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-13T22:22:51.679659507Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-13T22:22:52.170298236Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=info ts=2026-05-13T22:22:52.293670796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:22:52.302263981Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:22:52.73632962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:22:52.740856901Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-13T22:22:52.883837725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:22:55.439774649Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-13T22:22:55.439804094Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=2
level=debug ts=2026-05-13T22:22:55.896416708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-13T22:22:56.658457284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:22:56.66675309Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-13T22:22:57.884002428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:22:58.598464995Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-13T22:22:59.452067072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:22:59.457046217Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-13T22:23:00.058732561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:23:00.062171469Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-13T22:23:00.790056741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:23:00.799376784Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:23:01.813509264Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-13T22:23:01.813535234Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-13T22:23:02.884126727Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:23:02.8841546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:23:02.88416555Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:23:02.884187003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:23:02.932322438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:23:02.935990408Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
ts=2026-05-13T22:23:04.716723126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:23:05.026670895Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-13T22:23:07.64753393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:23:07.652173808Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-13T22:23:07.883389043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:23:08.241179996Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
ts=2026-05-13T22:23:08.348799699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35820"
level=debug ts=2026-05-13T22:23:11.455644332Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-13T22:23:12.883314723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:23:13.327626534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:23:13.332533384Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:23:14.598496277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:23:14.603518847Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-13T22:23:14.669984943Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-13T22:23:16.521838072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:23:16.525476403Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T22:23:17.883657979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:23:17.883796614Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-13T22:23:20.188653436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:23:20.193126468Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-13T22:23:21.098765142Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-13T22:23:21.809509423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:23:21.816073432Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-13T22:23:22.883798843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:23:23.857669998Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:23:24.312629807Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-13T22:23:25.585125225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:23:25.589039258Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:23:26.106424238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:23:26.110874329Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-13T22:23:26.832889127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:23:26.837417374Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:23:27.522111802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-13T22:23:27.52513116Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e2318a1e3%3A19e233176c7%3Ae56467ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=debug ts=2026-05-13T22:23:27.52720843Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-13T22:23:27.88359143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:23:28.989625228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T22:23:28.993932999Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e2318477d%3A19e236b1635%3Ae109ddfe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T22:23:29.380979067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:23:29.533287473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9173a55ce9d79e07 immediate=false num_chunks=1 total_comp="837 kB" avg_comp="837 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"build-e2e-rvxv\"}"
level=error ts=2026-05-13T22:23:29.539648898Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9173a55ce9d79e07/19e2322d3fd%3A19e2333e7c8%3A80eb53d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rvxv\"}"
level=debug ts=2026-05-13T22:23:30.741058845Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-13T22:23:31.483418167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:23:31.488233594Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-13T22:23:32.814006907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:23:32.817811989Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:23:32.864625151Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-13T22:23:32.86499677Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:23:32.883265899Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:23:32.883281882Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:23:32.883289421Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:23:32.883285876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:23:33.955377564Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-13T22:23:34.078157055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:23:34.081856818Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-13T22:23:34.719908992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:23:37.170008767Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-13T22:23:37.170050547Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-13T22:23:37.883915746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:23:38.628937666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:23:38.636398638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:23:39.235066842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:23:39.238594321Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:23:40.385985967Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-13T22:23:40.386008786Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=warn ts=2026-05-13T22:23:42.883222361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:23:43.598258037Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-13T22:23:44.205651581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:23:44.209969733Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=info ts=2026-05-13T22:23:44.561428888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:23:44.569644669Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:23:44.56966486Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-13T22:23:44.569699346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:23:44.577720598Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:23:46.247148307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:23:46.251230381Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:23:46.813212202Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-13T22:23:47.88363577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:23:49.668696256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:23:49.677227493Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:23:50.027136926Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-13T22:23:50.952978676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:23:50.956777169Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-13T22:23:52.626614922Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:23:52.884075429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:23:53.241595334Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-13T22:23:54.613722775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:23:54.62253858Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-13T22:23:56.45579256Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-13T22:23:57.574127453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:23:57.578124188Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-13T22:23:57.883460469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:23:59.670345796Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=info ts=2026-05-13T22:24:02.825623053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="578 B" avg_comp="578 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:24:02.833449369Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-13T22:24:02.883129334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-13T22:24:02.883152227Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:24:02.883160153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-13T22:24:02.883190955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:24:02.884307433Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=debug ts=2026-05-13T22:24:02.884322133Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-13T22:24:03.987010401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:24:03.991944599Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-13T22:24:04.72316444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=info ts=2026-05-13T22:24:04.814845259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:24:04.819446941Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-05-13T22:24:05.992176274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33584"
level=debug ts=2026-05-13T22:24:06.098286738Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-05-13T22:24:07.690027415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:24:07.694630522Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T22:24:07.883758621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:24:07.925884769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:24:07.930644464Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-13T22:24:08.353620715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44484"
level=debug ts=2026-05-13T22:24:09.312831225Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-13T22:24:10.698743283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:24:10.702094733Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-13T22:24:11.032987125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:24:11.036307462Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-13T22:24:12.527571258Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=warn ts=2026-05-13T22:24:12.884009196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:24:15.18328291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:24:15.189488741Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-13T22:24:15.741010408Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-13T22:24:15.960901786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:24:15.964720134Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-13T22:24:17.883530453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:24:18.447460911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T22:24:18.451848985Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e2318477d%3A19e236b1635%3Ae109ddfe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-13T22:24:18.954860639Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-13T22:24:20.348040704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9173a55ce9d79e07 immediate=false num_chunks=1 total_comp="837 kB" avg_comp="837 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"build-e2e-rvxv\"}"
level=error ts=2026-05-13T22:24:20.353371957Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9173a55ce9d79e07/19e2322d3fd%3A19e2333e7c8%3A80eb53d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rvxv\"}"
level=info ts=2026-05-13T22:24:21.846349607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:24:21.850036467Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-13T22:24:22.17078551Z caller=checkpoint.go:372 msg="writing series" size="620 kB"
level=debug ts=2026-05-13T22:24:22.170811594Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-13T22:24:22.883756136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:24:23.633136202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-13T22:24:23.636922002Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e2318a1e3%3A19e233176c7%3Ae56467ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-05-13T22:24:23.70215842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:24:23.705588781Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:24:25.384599545Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-13T22:24:26.942923896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:24:26.946218188Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-13T22:24:27.192627027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:24:27.690145657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88f8733267db3f3e immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="331 kB" avg_uncomp="331 kB" full=1 labels="{stream=\"stat-rep-rbyo\"}"
level=error ts=2026-05-13T22:24:27.693480508Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88f8733267db3f3e/19e2321c674%3A19e2329a835%3A4c741ccf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-rbyo\"}"
level=debug ts=2026-05-13T22:24:27.863352911Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-13T22:24:27.883579386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:24:27.909669417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:24:27.912821898Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:24:28.598278601Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-13T22:24:29.033604032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:24:29.04144031Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-13T22:24:31.14699524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:24:31.151593172Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-13T22:24:31.814911647Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-13T22:24:31.814933037Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
level=info ts=2026-05-13T22:24:32.46035965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:24:32.467348165Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-13T22:24:32.864352661Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-13T22:24:32.865403867Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-13T22:24:32.883783023Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:24:32.88378566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:24:32.883799825Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:24:32.883807923Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T22:24:34.189666618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="156 kB" avg_uncomp="156 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-13T22:24:34.194979564Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e23184b60%3A19e2318c332%3Aaac8c8e9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-13T22:24:34.725771818Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-gh25g-2661cecf 10.130.0.68:7946"
level=debug ts=2026-05-13T22:24:35.027171611Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-13T22:24:35.99495263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48040"
level=warn ts=2026-05-13T22:24:37.883434732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:24:38.241303558Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=debug ts=2026-05-13T22:24:41.455688711Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-13T22:24:42.883395178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:24:43.254988038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.7 MB" total_uncomp="23 MB" avg_uncomp="11 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-13T22:24:43.26381232Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e23184daa%3A19e233126d6%3Ab38f6b99?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-13T22:24:44.669890583Z caller=checkpoint.go:372 msg="writing series" size="624 kB"
level=info ts=2026-05-13T22:24:44.993548933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="437 kB" avg_comp="437 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-13T22:24:44.998335418Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e23184469%3A19e23621d55%3A241591b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-13T22:24:45.12272742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="596 B" avg_comp="596 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-13T22:24:45.126044413Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e2318a1e3%3A19e2318a1e4%3Ab82d8c86?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-13T22:24:46.153407564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.69:33008"
level=warn ts=2026-05-13T22:24:47.883407199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:24:47.88345093Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-13T22:24:51.098725918Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-13T22:24:52.765066935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="463 kB" avg_comp="463 kB" total_uncomp="2.9 MB" avg_uncomp="2.9 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-13T22:24:52.770909206Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e231845d0%3A19e235c8b8d%3Aab7699ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-13T22:24:52.883367537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:24:53.639712036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-13T22:24:53.643431045Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e2318a1e3%3A19e232ba404%3A63abb44c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-13T22:24:54.312448272Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-13T22:24:54.312473934Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-13T22:24:56.178067098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-13T22:24:56.181966152Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e23184756%3A19e23302a0d%3A95871574?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-13T22:24:56.514855132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-13T22:24:56.518245644Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e2318a1e3%3A19e2318a1e4%3A601912fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-13T22:24:57.527118998Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-13T22:24:57.88363707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:24:58.381700012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="448 kB" avg_comp="448 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-13T22:24:58.386294248Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e231844fd%3A19e23636355%3Adb836d31?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-13T22:25:00.742332258Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-13T22:25:00.742356438Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=debug ts=2026-05-13T22:25:01.261809711Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-13T22:25:02.065494679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-13T22:25:02.883975695Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-13T22:25:02.883991091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:25:02.884002465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-13T22:25:02.88402852Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-13T22:25:02.9843847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9173a55ce9d79e07 immediate=false num_chunks=1 total_comp="837 kB" avg_comp="837 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"build-e2e-rvxv\"}"
level=error ts=2026-05-13T22:25:02.989306301Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9173a55ce9d79e07/19e2322d3fd%3A19e2333e7c8%3A80eb53d2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rvxv\"}"
level=info ts=2026-05-13T22:25:03.831679555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-13T22:25:03.835508239Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e232289c4%3A19e23242005%3A88a169cf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-13T22:25:03.955566172Z caller=checkpoint.go:372 msg="writing series" size="359 kB"
level=info ts=2026-05-13T22:25:04.369513176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="22 kB" avg_uncomp="22 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-05-13T22:25:04.374653708Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/122c7b55617f1a36/19e2318a1e3%3A19e233176c7%3Ae56467ed?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
ts=2026-05-13T22:25:04.728777007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-ch27z-8f75872c 10.130.0.67:7946"
level=debug ts=2026-05-13T22:25:07.170117951Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-13T22:25:07.883341832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:25:10.384396262Z caller=checkpoint.go:372 msg="writing series" size="485 kB"
level=info ts=2026-05-13T22:25:10.701247508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-13T22:25:10.705270453Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e23184976%3A19e2318c332%3Ae41417d3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-13T22:25:10.849547335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="11 MB" avg_uncomp="11 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-13T22:25:10.855263054Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e231845a9%3A19e233d0155%3A16ed1255?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-13T22:25:11.764971389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-13T22:25:11.769453115Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e2318477d%3A19e236b1635%3Ae109ddfe?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-13T22:25:12.083563601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="677 kB" avg_comp="677 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-13T22:25:12.087939037Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e2318477d%3A19e235c6ec8%3Af290737b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-13T22:25:12.88342766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-13T22:25:13.598032404Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-13T22:25:13.598054685Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-13T22:25:13.619991105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-13T22:25:13.623394636Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e2318ada5%3A19e2318c332%3A29b296fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-13T22:25:13.754470972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="157 kB" avg_uncomp="157 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-13T22:25:13.758056077Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e231848fa%3A19e2318a1e4%3Ad4f22578?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-13T22:25:14.394086997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="18 MB" avg_comp="2.2 MB" total_uncomp="153 MB" avg_uncomp="19 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-13T22:25:14.401453568Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e23184a9c%3A19e2318b94f%3A8037b345?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-13T22:25:16.423536263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-05-13T22:25:16.42717212Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1da2f4dc67213293/19e2318a1e3%3A19e2318a1e4%3Adc6524b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=debug ts=2026-05-13T22:25:16.812343887Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=warn ts=2026-05-13T22:25:17.883803063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-13T22:25:18.669731864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-13T22:25:18.67324271Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e2318a1e3%3A19e2318c332%3A507b4b1c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-13T22:25:20.02682362Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-13T22:25:22.206333182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="469 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-13T22:25:22.213851401Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e23184472%3A19e2321c1bf%3A1fbeeeeb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"