level=info ts=2026-05-07T14:55:09.957058939Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-07T14:55:09.95710813Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-07T14:55:09.957147132Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-07T14:55:09.958905508Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-07T14:55:09.95932318Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:55:09.95939191Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.82 inf=eth0
level=error ts=2026-05-07T14:55:09.965163449Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T14:55:09.965186994Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-07T14:55:09.966092384Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-07T14:55:09.966126606Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-07T14:55:09.966865183Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-07T14:55:09.966916618Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-07T14:55:09.966894801Z
level=debug ts=2026-05-07T14:55:09.966994204Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-07T14:55:09.967055163Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-07T14:55:09.96710242Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-07T14:55:09.967112521Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-07T14:55:09.967110662Z
level=debug ts=2026-05-07T14:55:09.967121375Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-07T14:55:09.968327229Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-66b5a331
ts=2026-05-07T14:55:09.96837326Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-07T14:55:09.968386289Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.82 advertisePort=7946
level=debug ts=2026-05-07T14:55:09.968407705Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.82 advertisePort=7946
level=info ts=2026-05-07T14:55:09.968543792Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-05-07T14:55:09.969325051Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-07T14:55:09.969408039Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.82 inf=eth0
ts=2026-05-07T14:55:09.972274009Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T14:55:09.972290589Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:55:09.972303302Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=3.760853ms
level=error ts=2026-05-07T14:55:09.972312331Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-07T14:55:09.972323227Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-05-07T14:55:09.974865374Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:55:09.974881197Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T14:55:09.983934107Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-05-07T14:55:09.983985259Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-07T14:55:09.984047201Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-05-07T14:55:09.98405975Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-05-07T14:55:09.984070505Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-07T14:55:09.984074384Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-07T14:55:09.984081579Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-07T14:55:09.984089854Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-07T14:55:09.984093859Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-07T14:55:09.984101104Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-07T14:55:09.984103995Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-07T14:55:09.98450633Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-07T14:55:09.984514288Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-05-07T14:55:09.984514929Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-05-07T14:55:09.984527289Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-05-07T14:55:09.984540925Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-07T14:55:09.984564561Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-07T14:55:09.984568486Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-07T14:55:09.984571075Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-07T14:55:09.98468061Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-07T14:55:09.984685371Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-07T14:55:09.984689068Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-07T14:55:09.984692Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-07T14:55:09.984694984Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-07T14:55:09.98469819Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-07T14:55:09.98470815Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-07T14:55:09.984718261Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T14:55:09.984764937Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=23.402µs success=true
level=info ts=2026-05-07T14:55:09.984777196Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-07T14:55:09.984838009Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=130.773µs errors=false
level=info ts=2026-05-07T14:55:09.984851592Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-07T14:55:09.985221574Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=514.196µs errors=false
level=info ts=2026-05-07T14:55:09.985234097Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-07T14:55:09.985247545Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=532.859µs
level=info ts=2026-05-07T14:55:09.985281723Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-07T14:55:09.985373481Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-07T14:55:09.985371379Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=20.860445762s
level=info ts=2026-05-07T14:55:09.985393334Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-07T14:55:09.985417308Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-07T14:55:09.985454161Z caller=loki.go:615 msg="Loki started" startup_time=86.619944ms
level=debug ts=2026-05-07T14:55:09.985766748Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-07T14:55:09.98577877Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-07T14:55:09.985834303Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-07T14:55:11.592052243Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:55:11.592076925Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:55:14.98652586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:55:15.348569651Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:55:15.348591074Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:55:19.985484186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:55:21.148890325Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T14:55:21.14891548Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:55:24.986210032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:55:27.734407878Z caller=logging.go:137 msg="GET /ready (503) 103.729µs"
level=warn ts=2026-05-07T14:55:29.986160429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:55:34.062137597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.83:7946"
ts=2026-05-07T14:55:34.063711838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.81:7946"
ts=2026-05-07T14:55:34.065801319Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.65:7946: dial tcp 10.128.0.65:7946: connect: connection refused"
ts=2026-05-07T14:55:34.066968774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.66:7946"
ts=2026-05-07T14:55:34.069522547Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.83:7946: dial tcp 10.130.0.83:7946: connect: connection refused"
level=info ts=2026-05-07T14:55:34.069533747Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.097206596s
level=warn ts=2026-05-07T14:55:34.985652918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:55:37.73347352Z caller=logging.go:137 msg="GET /ready (503) 70.601µs"
level=warn ts=2026-05-07T14:55:39.986207467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:55:39.986191428Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T14:55:39.986229147Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-07T14:55:39.986237649Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:55:44.985772153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:55:49.98608425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:55:50.097833505Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T14:55:54.986310059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:55:59.985829512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:56:02.554938709Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T14:56:02.555027775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:56:02.56140772Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T14:56:04.986160567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:56:08.462465598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-07T14:56:09.966211812Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T14:56:09.966724277Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T14:56:09.986219464Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:56:09.986238849Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T14:56:09.986230731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:56:09.986246882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:56:14.986313314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:56:16.613627315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:56:16.623232994Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T14:56:19.986180386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:56:20.100596093Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T14:56:24.986055742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:56:28.166587867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-07T14:56:29.358915437Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T14:56:29.985950016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:56:34.87039886Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:32954"
level=warn ts=2026-05-07T14:56:34.985888233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:56:38.081644642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:56:38.088192266Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T14:56:39.985503667Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T14:56:39.985500953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:56:39.985526878Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T14:56:39.985535172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:56:44.985873311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:56:45.559056493Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:56:49.986150052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:56:50.103623215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T14:56:54.985524959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:56:59.0165648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37190"
level=warn ts=2026-05-07T14:56:59.985521722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:57:03.196221793Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T14:57:04.986429385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:57:09.966373868Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T14:57:09.966908696Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T14:57:09.986290662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:57:09.986296098Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:57:09.986314518Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T14:57:09.986322645Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T14:57:13.979243549Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:57:14.985496706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:57:19.986468529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:57:20.10604377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=warn ts=2026-05-07T14:57:24.986099368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:57:26.491747213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:57:26.497864728Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T14:57:26.73184536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53938"
level=debug ts=2026-05-07T14:57:28.498138933Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T14:57:29.017803388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48120"
level=warn ts=2026-05-07T14:57:29.985566806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:57:34.986122527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:57:39.277245653Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T14:57:39.279147473Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T14:57:39.985598184Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:57:39.985630862Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T14:57:39.985625494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:57:39.98564159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:57:44.985666271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:57:49.986314885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:57:50.108796964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T14:57:54.986295201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:57:59.018972637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45058"
level=warn ts=2026-05-07T14:57:59.986099516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:58:03.259912303Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T14:58:04.986502661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:58:09.966351694Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T14:58:09.966779059Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T14:58:09.986075441Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T14:58:09.986088951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:58:09.986108433Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T14:58:09.986117958Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T14:58:14.970817108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:58:14.977144633Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T14:58:14.986220794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:58:19.985608279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:58:20.110915662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T14:58:20.355114719Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:58:24.986100167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:58:26.73667372Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55956"
level=warn ts=2026-05-07T14:58:29.985926161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:58:34.985651869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:58:35.855099909Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T14:58:39.985580068Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:58:39.985608922Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T14:58:39.985607087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:58:39.985621192Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:58:44.98588831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:58:49.985927303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:58:50.113469534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T14:58:54.98599668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:58:59.024028437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38566"
level=warn ts=2026-05-07T14:58:59.98603756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:04.985748494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:59:05.613430811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T14:59:06.142883244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:59:06.14923594Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T14:59:09.966209865Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T14:59:09.966743665Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T14:59:09.986074173Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:59:09.986097129Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T14:59:09.986091273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T14:59:09.986104619Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:59:14.986367106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:59:17.342799484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T14:59:19.985663384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:59:20.11616578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T14:59:24.986489753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:59:29.025423056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37902"
level=warn ts=2026-05-07T14:59:29.986445646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:34.985778505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T14:59:38.384240493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T14:59:39.985853698Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T14:59:39.98587677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T14:59:39.985884508Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T14:59:39.985903954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:44.986105423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T14:59:49.985553966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:59:50.119086948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T14:59:54.985504718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T14:59:56.742569877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40410"
level=info ts=2026-05-07T14:59:59.909603443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T14:59:59.916942725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T14:59:59.985825846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:00:04.891125825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38632"
level=warn ts=2026-05-07T15:00:04.985939027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:07.813132711Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:00:09.966337959Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:00:09.966893294Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=debug ts=2026-05-07T15:00:09.968168058Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T15:00:09.968206958Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975739 n=1
level=debug ts=2026-05-07T15:00:09.968274441Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975739 n=1
level=info ts=2026-05-07T15:00:09.986197918Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:00:09.986214501Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:00:09.986233563Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:00:09.986241291Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:00:09.986272719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:00:09.986333518Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-07T15:00:09.98634128Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:00:09.986355956Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.951µs success=true
level=debug ts=2026-05-07T15:00:09.986545352Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=debug ts=2026-05-07T15:00:14.346463536Z caller=checkpoint.go:372 msg="writing series" size="5.0 MB"
level=debug ts=2026-05-07T15:00:14.346485688Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.1 MB" series=2
level=warn ts=2026-05-07T15:00:14.986352809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:18.696355568Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-07T15:00:19.985783008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:00:20.122273909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:00:23.051329779Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-07T15:00:24.986430059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:27.406147471Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
level=debug ts=2026-05-07T15:00:28.771145191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T15:00:29.986291425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:31.761160199Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-07T15:00:34.985942001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:36.116571186Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:00:39.941839145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:00:39.950946363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:00:39.98583795Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:00:39.985863291Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:00:39.985870952Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:00:39.985879962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:40.471130789Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-07T15:00:42.343324302Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:00:44.825837006Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-07T15:00:44.985784876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:00:49.180691319Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=debug ts=2026-05-07T15:00:49.180714008Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-07T15:00:49.985470031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:00:50.124124318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:00:53.535162486Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:00:54.986086214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:00:56.746915028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43672"
level=debug ts=2026-05-07T15:00:57.890247522Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-07T15:00:59.986089239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:02.24556221Z caller=checkpoint.go:372 msg="writing series" size="390 kB"
level=warn ts=2026-05-07T15:01:04.985727893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:06.600046377Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=info ts=2026-05-07T15:01:09.966180167Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:01:09.967003204Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:01:09.986350082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:01:09.986385526Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:01:09.986393396Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:01:09.98640698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:10.954916832Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-07T15:01:11.477593051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:01:14.985682027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:15.309744629Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=debug ts=2026-05-07T15:01:19.664674423Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-07T15:01:19.985543356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:01:20.126230821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:01:24.019523517Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-07T15:01:24.985717888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:01:26.747936239Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41030"
level=debug ts=2026-05-07T15:01:28.374703115Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=debug ts=2026-05-07T15:01:29.933868735Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-07T15:01:29.985925762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:32.728730265Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-07T15:01:34.98573708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:37.084233457Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=info ts=2026-05-07T15:01:39.000647871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:01:39.00946523Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:01:39.985868147Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:01:39.985895152Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:01:39.985902858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:01:39.985918741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:41.439043796Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-07T15:01:41.479318772Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:01:44.985488114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:45.79363111Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=warn ts=2026-05-07T15:01:49.986322622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:01:50.128994151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:01:50.148689936Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=debug ts=2026-05-07T15:01:54.503720796Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
level=debug ts=2026-05-07T15:01:54.503743121Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=15
level=warn ts=2026-05-07T15:01:54.985597731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:01:58.858189301Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-07T15:01:59.986296039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:03.21325014Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-05-07T15:02:04.986117509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:07.567548881Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-07T15:02:09.89403775Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:02:09.966983317Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:02:09.967938573Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:02:09.985752738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:02:09.985754981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:09.985777102Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:02:09.985784818Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:02:11.92259572Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-07T15:02:14.985994386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:16.277317314Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:02:19.986174834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:02:20.130680326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:02:20.632914198Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=warn ts=2026-05-07T15:02:24.986294071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:24.987375681Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
ts=2026-05-07T15:02:26.749944077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34832"
level=debug ts=2026-05-07T15:02:29.342108339Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-07T15:02:29.985892076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:33.696302479Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
ts=2026-05-07T15:02:34.905809573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:58350"
level=warn ts=2026-05-07T15:02:34.98644141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:02:35.665303624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:02:35.67440869Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:02:35.674428702Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:02:35.674451546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:02:35.680377391Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:02:35.832003187Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-07T15:02:35.833705101Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:02:36.660244529Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:02:38.051746935Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-07T15:02:39.985550508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:02:39.985559746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:39.985579017Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:02:39.985594274Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:02:42.406838212Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T15:02:44.986310807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:46.761396199Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-07T15:02:49.986167354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:02:50.132628831Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:02:51.115943858Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=info ts=2026-05-07T15:02:51.973364924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:02:51.98259379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:02:54.986005062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:02:55.471349311Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
ts=2026-05-07T15:02:56.75144226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58792"
ts=2026-05-07T15:02:59.038830527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45248"
level=debug ts=2026-05-07T15:02:59.825410049Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=warn ts=2026-05-07T15:02:59.986190654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:04.180542981Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
ts=2026-05-07T15:03:04.908889491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:46034"
level=warn ts=2026-05-07T15:03:04.985665987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:07.893025539Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:03:08.535927731Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=info ts=2026-05-07T15:03:09.96695245Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:03:09.967816088Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:03:09.98636756Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:03:09.986386842Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:03:09.986378848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:03:09.986394594Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:03:12.89037538Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-07T15:03:14.986453526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:17.245235651Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-07T15:03:19.986304447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:03:20.135793196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:03:21.600407431Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=debug ts=2026-05-07T15:03:21.600432166Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=20
level=info ts=2026-05-07T15:03:22.445576178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:03:22.454989916Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:03:24.986543394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:25.954379246Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
ts=2026-05-07T15:03:29.039970595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42592"
level=warn ts=2026-05-07T15:03:29.985919428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:30.3095569Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-07T15:03:32.284890315Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:03:33.454633446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:03:34.664322435Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-07T15:03:34.985956366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:39.01927658Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-07T15:03:39.985556257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:03:39.985579385Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:03:39.985587242Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:03:39.985583196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:43.373827402Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-07T15:03:44.985678002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:47.729451202Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
level=warn ts=2026-05-07T15:03:49.986300228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:03:50.138840564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:03:52.083396946Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-07T15:03:54.986117238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:03:56.438419574Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
ts=2026-05-07T15:03:56.753786899Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60378"
level=warn ts=2026-05-07T15:03:59.985950209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:00.793527461Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:04:04.98646861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:05.148902216Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-07T15:04:06.318384607Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:04:09.503833647Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
level=debug ts=2026-05-07T15:04:09.503856067Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-07T15:04:09.966812179Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:04:09.967897038Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:04:09.985570342Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:04:09.985564571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:09.985593864Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:04:09.98560115Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:04:13.858608298Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=warn ts=2026-05-07T15:04:14.986316311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:15.30808315Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:04:18.213355986Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=warn ts=2026-05-07T15:04:19.985561408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:04:20.141223938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T15:04:21.090143678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:04:21.100075267Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:04:22.568277424Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:04:22.568302602Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=warn ts=2026-05-07T15:04:24.985540098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:04:26.755418937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40108"
level=debug ts=2026-05-07T15:04:26.92260092Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-07T15:04:29.985476697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:31.277393949Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-07T15:04:31.539089785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:04:34.986297675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:35.632675939Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T15:04:39.985474044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:04:39.985482292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:04:39.985500319Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:04:39.985509921Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:04:39.986565588Z caller=checkpoint.go:384 msg="flushing series" totalSize="194 kB" series=3
level=info ts=2026-05-07T15:04:39.990401031Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-07T15:04:39.993709921Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00721406s
level=warn ts=2026-05-07T15:04:44.985881273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:04:49.985520598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:04:50.144089573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T15:04:54.985637322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:04:56.757391728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59452"
level=warn ts=2026-05-07T15:04:59.986127157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:02.179709533Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:05:02.179802862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:05:02.189399306Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:05:02.5901003Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:05:04.920644913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40532"
level=warn ts=2026-05-07T15:05:04.98650556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:05:09.966201065Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:05:09.967133044Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:05:09.985860371Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:05:09.98591331Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:05:09.985932517Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:05:09.985929996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:05:09.985939639Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:05:09.98602309Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-07T15:05:09.98603205Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:05:09.986049383Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.6µs success=true
level=debug ts=2026-05-07T15:05:09.986222983Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-07T15:05:12.439557465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:05:12.447941636Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:05:13.789095586Z caller=checkpoint.go:372 msg="writing series" size="357 B"
level=warn ts=2026-05-07T15:05:14.985854862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:17.592503028Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=info ts=2026-05-07T15:05:19.134390648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:05:19.144452781Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:05:19.98622664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:05:20.147493413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:05:21.398909184Z caller=checkpoint.go:372 msg="writing series" size="5.8 MB"
level=debug ts=2026-05-07T15:05:21.39893184Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.8 MB" series=4
level=warn ts=2026-05-07T15:05:24.985691823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:25.198331659Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-05-07T15:05:26.462384188Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:05:29.000493616Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T15:05:29.045048094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46018"
level=warn ts=2026-05-07T15:05:29.98551666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:32.804297578Z caller=checkpoint.go:372 msg="writing series" size="547 kB"
level=warn ts=2026-05-07T15:05:34.985639478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:36.606711376Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-07T15:05:39.985820542Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:05:39.985847969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:05:39.985855731Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:05:39.98584727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:40.409334564Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-07T15:05:44.211733314Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=warn ts=2026-05-07T15:05:44.985612656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:48.015272283Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-07T15:05:49.986072625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:05:50.150513097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:05:51.817798757Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-07T15:05:52.057596566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:05:52.068995352Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:05:54.985833166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:05:55.620494672Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
ts=2026-05-07T15:05:56.76215131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36326"
level=debug ts=2026-05-07T15:05:58.120954398Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:05:58.921220747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:05:58.928557872Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:05:59.423307347Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:05:59.98621943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:01.758530127Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:06:03.226424942Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:06:04.986166703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:07.029110797Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-07T15:06:09.966185458Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:06:09.966567058Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:06:09.985896181Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:06:09.985914766Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:06:09.985922546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:06:09.985913793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:10.831780812Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-07T15:06:14.642660085Z caller=checkpoint.go:372 msg="writing series" size="8.5 MB"
level=debug ts=2026-05-07T15:06:14.642684345Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.4 MB" series=14
level=warn ts=2026-05-07T15:06:14.986311064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:18.437742394Z caller=checkpoint.go:372 msg="writing series" size="392 kB"
level=warn ts=2026-05-07T15:06:19.98622911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:06:20.152793391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:06:22.240470814Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-07T15:06:23.487220873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:06:24.986445678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:26.043222205Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-07T15:06:29.845720694Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-07T15:06:29.98613758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:33.649310564Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-07T15:06:34.985593725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:37.451286218Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-07T15:06:39.98635831Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:06:39.986369536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:39.986388068Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:06:39.986397215Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:06:41.254753926Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-07T15:06:44.026417902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:06:44.039775364Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:06:44.76047734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:06:44.767583378Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:06:44.985712831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:45.056987062Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=debug ts=2026-05-07T15:06:48.860135042Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-07T15:06:49.986428226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:06:50.155240735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:06:52.662584493Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-07T15:06:53.953079773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:06:54.986304069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:06:56.465313998Z caller=checkpoint.go:372 msg="writing series" size="513 B"
ts=2026-05-07T15:06:56.764861322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58956"
level=warn ts=2026-05-07T15:06:59.986381004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:00.26827802Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-05-07T15:07:00.734440294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:07:00.737090975Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:07:04.071491304Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=debug ts=2026-05-07T15:07:04.071514811Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
ts=2026-05-07T15:07:04.93590164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56552"
level=warn ts=2026-05-07T15:07:04.985542214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:07.874113518Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-07T15:07:09.966512273Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:07:09.971137237Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:07:09.985426354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:07:09.985462174Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:07:09.985474932Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:07:09.985537092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:11.67733418Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=warn ts=2026-05-07T15:07:14.986401723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:15.479373903Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-07T15:07:19.299400337Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-07T15:07:19.779719445Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:07:19.986310375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:07:20.15696028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:07:23.085646852Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:07:24.986068273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:26.888701507Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=warn ts=2026-05-07T15:07:29.986432954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:30.690785103Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-07T15:07:34.49436957Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=warn ts=2026-05-07T15:07:34.986299952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:38.082544861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:07:38.296154115Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T15:07:39.427103084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:07:39.438314069Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:07:39.986204707Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:07:39.986229643Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:07:39.986223959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:07:39.986237265Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:07:42.099825616Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-07T15:07:44.103326598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:07:44.11116033Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:07:44.986501579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:07:45.902613514Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=debug ts=2026-05-07T15:07:48.923207134Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:07:49.704761652Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=debug ts=2026-05-07T15:07:49.704797161Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-05-07T15:07:49.985491274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:07:50.158240694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:07:53.507617807Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=warn ts=2026-05-07T15:07:54.985980018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:07:56.767459202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46686"
level=debug ts=2026-05-07T15:07:57.31048004Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-07T15:07:59.986236674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:01.113597749Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-07T15:08:04.916616326Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
ts=2026-05-07T15:08:04.941778729Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57200"
level=warn ts=2026-05-07T15:08:04.986298137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:08.719437024Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-07T15:08:09.966209334Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:08:09.968736934Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:08:09.986050884Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:08:09.986074841Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:08:09.986068328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:08:09.986086058Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:08:12.522587351Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-07T15:08:14.986008684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:16.324756646Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:08:19.384471999Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:08:19.987468039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:20.127996381Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
ts=2026-05-07T15:08:20.159422724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:08:23.930414979Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=warn ts=2026-05-07T15:08:24.986452748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:27.732800369Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:08:29.985532078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:08:30.78544016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="96 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:08:30.797072596Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:08:31.535976268Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-07T15:08:34.986546367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:35.339383549Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=debug ts=2026-05-07T15:08:35.339405453Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-05-07T15:08:36.245526626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:08:39.141490481Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-07T15:08:39.834684003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:08:39.841533468Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:08:39.986205484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:08:39.986200387Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:08:39.986232806Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:08:39.986242079Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:08:42.944731225Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-07T15:08:44.985772502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:46.74744463Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-07T15:08:47.952738909Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:08:49.986027733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:08:50.161743838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:08:50.549746191Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-05-07T15:08:54.352915367Z caller=checkpoint.go:372 msg="writing series" size="318 kB"
level=warn ts=2026-05-07T15:08:54.98609234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:08:58.156187365Z caller=checkpoint.go:372 msg="writing series" size="348 kB"
level=debug ts=2026-05-07T15:08:58.156211247Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-07T15:08:59.98647253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:01.958921293Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-07T15:09:04.98629145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:05.761113548Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=debug ts=2026-05-07T15:09:09.564051559Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=info ts=2026-05-07T15:09:09.96676464Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:09:09.967889951Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:09:09.985617944Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:09:09.985640072Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:09:09.985629344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:09:09.985648655Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:09:13.367136129Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-05-07T15:09:14.778273498Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:09:14.985653654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:17.169904055Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-07T15:09:19.986195451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:09:20.164931861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:09:20.354148868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:09:20.363430165Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:09:20.974205055Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-07T15:09:20.974294767Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=debug ts=2026-05-07T15:09:24.775794267Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-07T15:09:24.985719316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:09:26.773387154Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35352"
level=debug ts=2026-05-07T15:09:28.578882591Z caller=checkpoint.go:372 msg="writing series" size="438 kB"
level=info ts=2026-05-07T15:09:29.45227318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:09:29.459672278Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:09:29.459696598Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:09:29.459721503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:09:29.46591278Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:09:29.986321187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:32.381502284Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-07T15:09:34.985921962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:36.184101141Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-07T15:09:38.751558952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:09:39.985428985Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:09:39.985459152Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:09:39.985467989Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:09:39.98549613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:39.986489784Z caller=checkpoint.go:384 msg="flushing series" totalSize="451 kB" series=4
level=info ts=2026-05-07T15:09:40.089410983Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-07T15:09:40.093420463Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.107239495s
level=info ts=2026-05-07T15:09:41.55355057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:09:41.562224157Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:09:44.986226161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:09:46.84262714Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:09:49.986150649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:09:50.167426278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T15:09:54.985543899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:09:56.77471339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60178"
level=warn ts=2026-05-07T15:09:59.985980402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:10:04.985873742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:10:07.656659691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:10:07.67041849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:10:09.966248765Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:10:09.966573781Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:10:09.986164956Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T15:10:09.986220983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:10:09.986233204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:10:09.986247494Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:10:09.986267016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:10:09.986320144Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-07T15:10:09.986329372Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:10:09.986347345Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.247µs success=true
level=debug ts=2026-05-07T15:10:09.986527114Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-07T15:10:13.405096627Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-07T15:10:14.985876941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:10:16.704999752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:10:16.712460639Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:10:16.822785796Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-07T15:10:19.985837714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:10:20.169787591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:10:20.244285238Z caller=checkpoint.go:372 msg="writing series" size="6.4 MB"
level=debug ts=2026-05-07T15:10:20.24430653Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=4
level=debug ts=2026-05-07T15:10:22.675899794Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:10:23.657830549Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-07T15:10:24.985684297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:27.075292261Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-07T15:10:29.986055054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:30.494298643Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=debug ts=2026-05-07T15:10:33.910942854Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
ts=2026-05-07T15:10:34.957364302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33724"
level=warn ts=2026-05-07T15:10:34.985791226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:36.134681286Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:10:37.32920216Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:10:39.985485699Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:10:39.985489848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:39.98551075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:10:39.985519002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:10:40.746423216Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=debug ts=2026-05-07T15:10:44.164715193Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-07T15:10:44.985662425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:10:47.58173447Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-07T15:10:49.98552005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:10:50.172110206Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:10:50.901264972Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:10:50.999687369Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=debug ts=2026-05-07T15:10:50.999709104Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-07T15:10:54.417282281Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:10:54.986229998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:10:56.777234823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58744"
level=debug ts=2026-05-07T15:10:57.835409915Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T15:10:59.977272011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.2 MB" avg_comp="2.6 MB" total_uncomp="45 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:10:59.985506521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-07T15:10:59.986332981Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:11:00.992804922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:11:00.999849458Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:11:01.252524086Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T15:11:04.670107176Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:11:04.986414657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:08.088406342Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-05-07T15:11:09.966509106Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:11:09.967006684Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:11:09.985913054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:11:09.985914879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:09.985934314Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:11:09.985942018Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:11:11.526450041Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-07T15:11:11.52648307Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=6
level=debug ts=2026-05-07T15:11:14.9234683Z caller=checkpoint.go:372 msg="writing series" size="346 kB"
level=warn ts=2026-05-07T15:11:14.986019548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:18.341711295Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-07T15:11:19.985993697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:11:20.172919473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:11:20.843375887Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:11:21.759100642Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:11:24.985927799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:25.177292839Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=debug ts=2026-05-07T15:11:25.543486739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:11:25.545359514Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:11:28.594340835Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
ts=2026-05-07T15:11:29.07283035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50986"
level=warn ts=2026-05-07T15:11:29.98629405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:32.012917115Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-07T15:11:34.985734041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:35.429823549Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=debug ts=2026-05-07T15:11:38.847971102Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-07T15:11:39.985742635Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:11:39.985768992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:11:39.985759319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:11:39.985777811Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:11:42.265341045Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-07T15:11:44.985725729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:45.68285689Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-07T15:11:49.101272878Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-07T15:11:49.294875904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:11:49.30339561Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:11:49.986417772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:11:50.174473389Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:11:52.519221223Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=warn ts=2026-05-07T15:11:54.986511847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:11:55.706232828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:11:55.936897425Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
ts=2026-05-07T15:11:56.779430855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42442"
level=debug ts=2026-05-07T15:11:59.354198725Z caller=checkpoint.go:372 msg="writing series" size="652 kB"
level=debug ts=2026-05-07T15:11:59.354227215Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=14
level=info ts=2026-05-07T15:11:59.35813599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:11:59.368318512Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:11:59.368343851Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:11:59.368375846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:11:59.378603208Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:11:59.986245112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:02.771515436Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-07T15:12:03.159446718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:12:04.986473349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:06.189885199Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-07T15:12:09.60766451Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:12:09.96638864Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:12:09.966471515Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T15:12:09.985475015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:12:09.985488651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:12:09.985510094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:12:09.985518153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:12:13.024841687Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-07T15:12:14.98591722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:16.442857728Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-07T15:12:16.462102773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:12:16.470459432Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:12:19.861079753Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=warn ts=2026-05-07T15:12:19.986302662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:12:20.176578155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:12:23.278404102Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=debug ts=2026-05-07T15:12:23.436240274Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:12:24.985982638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:26.69639871Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=debug ts=2026-05-07T15:12:26.69642455Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
ts=2026-05-07T15:12:26.780481692Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:39184"
level=warn ts=2026-05-07T15:12:29.986014873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:30.113338963Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T15:12:33.108836774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:12:33.116726793Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:12:33.53184752Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
ts=2026-05-07T15:12:34.986064142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:44712"
level=warn ts=2026-05-07T15:12:34.986086923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:36.949241939Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=info ts=2026-05-07T15:12:39.985444852Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:12:39.985473805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:12:39.985465065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:12:39.985483088Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:12:40.366554193Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-07T15:12:43.784658028Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-07T15:12:44.986113717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:47.202591357Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
level=debug ts=2026-05-07T15:12:49.882211339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:12:49.985561095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:12:50.178611741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:12:50.271090952Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:12:50.62052957Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-07T15:12:51.065106328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:12:51.072436515Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:12:54.038218435Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-07T15:12:54.985851155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:12:57.45490489Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-07T15:12:59.986315546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:00.873342637Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-07T15:13:00.873367311Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=debug ts=2026-05-07T15:13:04.291444093Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-07T15:13:04.986052246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:07.70864665Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-07T15:13:09.966947758Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:13:09.968158768Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:13:09.985789923Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:13:09.985785317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:09.985813167Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:13:09.985824143Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:13:11.125970043Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:13:14.544343951Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=warn ts=2026-05-07T15:13:14.986194484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:17.962248557Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-05-07T15:13:19.985961429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:13:20.18016726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:13:20.198876612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:13:20.209656856Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:13:21.379936354Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-07T15:13:22.117009018Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:13:24.797279873Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-07T15:13:24.98565589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:13:26.782504564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48810"
level=debug ts=2026-05-07T15:13:28.215430751Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=warn ts=2026-05-07T15:13:29.985555141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:31.398469991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:13:31.632206267Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-07T15:13:34.985857915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:35.050231639Z caller=checkpoint.go:372 msg="writing series" size="384 kB"
level=debug ts=2026-05-07T15:13:35.050286481Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=debug ts=2026-05-07T15:13:38.467931931Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-07T15:13:39.986324907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:13:39.986387661Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:13:39.986399918Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:13:39.986401814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:13:40.142826488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="165 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:13:40.150134927Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:13:41.886090688Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=warn ts=2026-05-07T15:13:44.985756708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:45.303510438Z caller=checkpoint.go:372 msg="writing series" size="361 kB"
level=debug ts=2026-05-07T15:13:48.721059242Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-07T15:13:49.985576308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:13:50.181627421Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:13:52.138752941Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=warn ts=2026-05-07T15:13:54.985895096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:13:55.366632127Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:13:55.557641534Z caller=checkpoint.go:372 msg="writing series" size="799 kB"
level=debug ts=2026-05-07T15:13:55.557664096Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=6
level=debug ts=2026-05-07T15:13:58.974429011Z caller=checkpoint.go:372 msg="writing series" size="777 kB"
level=warn ts=2026-05-07T15:13:59.986094167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:02.392739552Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=warn ts=2026-05-07T15:14:04.985967516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:05.80994133Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=debug ts=2026-05-07T15:14:09.227173916Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=debug ts=2026-05-07T15:14:09.227197612Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-07T15:14:09.966718485Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:14:09.967331015Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:14:09.986065302Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:14:09.986097551Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:14:09.986107404Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:14:09.98607389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:14:10.362307724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:14:10.373888763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:14:12.645604206Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=warn ts=2026-05-07T15:14:14.98575906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:16.06364945Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
level=debug ts=2026-05-07T15:14:19.483987169Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-07T15:14:19.484012102Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=3
level=warn ts=2026-05-07T15:14:19.985595027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:14:20.184001849Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:14:22.898587693Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-07T15:14:24.986157031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:26.316805657Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
ts=2026-05-07T15:14:26.786924439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59824"
level=debug ts=2026-05-07T15:14:28.344483347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:14:29.578235014Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:14:29.733565391Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=warn ts=2026-05-07T15:14:29.986205517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:33.151325917Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-07T15:14:34.986781958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:36.569322535Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T15:14:39.627080683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="188 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:14:39.639941286Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:14:39.986290235Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:14:39.986315015Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:14:39.986322215Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:14:39.986328559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:14:39.987355712Z caller=checkpoint.go:384 msg="flushing series" totalSize="754 kB" series=5
level=info ts=2026-05-07T15:14:39.992329176Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-07T15:14:39.99430355Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007833234s
level=warn ts=2026-05-07T15:14:44.986329517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:14:49.986067927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:14:50.18674708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T15:14:54.986206398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:14:58.313958198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:14:58.321656918Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:14:58.505396582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:14:58.5054581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:14:58.511788388Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:14:59.985944504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:15:04.986370009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:15:09.966187622Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:15:09.968501166Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T15:15:09.968549559Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975740 n=1
level=debug ts=2026-05-07T15:15:09.968595051Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975740 n=1
level=debug ts=2026-05-07T15:15:09.968856653Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:15:09.985482303Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:15:09.985534899Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:15:09.985557192Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:15:09.985565059Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:15:09.985559163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:15:09.985613451Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-07T15:15:09.98562386Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:15:09.985643359Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.882µs success=true
level=debug ts=2026-05-07T15:15:09.985865147Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-07T15:15:13.362029216Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:15:13.36205091Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=info ts=2026-05-07T15:15:14.544672461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:15:14.551839119Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:15:14.986434311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:16.736130329Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-07T15:15:19.985792944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:20.115279654Z caller=checkpoint.go:372 msg="writing series" size="7.1 MB"
level=debug ts=2026-05-07T15:15:20.115304418Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.1 MB" series=2
ts=2026-05-07T15:15:20.18914495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:15:23.486769797Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=debug ts=2026-05-07T15:15:23.953742937Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:15:23.955718914Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T15:15:24.985852803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:26.861048946Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-07T15:15:29.985555054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:30.236357737Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=debug ts=2026-05-07T15:15:32.880301311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:15:33.611235521Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:15:34.985746367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:36.986144098Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:15:38.691191426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="212 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:15:38.7054633Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:15:39.985999894Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:15:39.986031863Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:15:39.986041083Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:15:39.98604219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:40.361431193Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-07T15:15:40.569450711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:15:40.576300525Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:15:43.571095404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:15:43.578496262Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:15:43.578519927Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:15:43.578546637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:15:43.584932833Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:15:43.736311153Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-07T15:15:44.9859226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:15:47.11135149Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-07T15:15:49.985829734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:15:50.190546777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:15:50.486151878Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-07T15:15:50.48617618Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-07T15:15:53.861682428Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:15:54.98550971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:15:56.792483732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:59572"
level=debug ts=2026-05-07T15:15:57.236102038Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-07T15:15:59.985528948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:00.611376752Z caller=checkpoint.go:372 msg="writing series" size="285 kB"
level=debug ts=2026-05-07T15:16:01.459664037Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:16:03.112158852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:16:03.120121047Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:16:03.986724588Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:16:04.985697061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:07.361421772Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-07T15:16:09.96641924Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:16:09.966457632Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:16:09.986091691Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:16:09.986121152Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:16:09.986112322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:16:09.986129054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:16:10.769287307Z caller=checkpoint.go:372 msg="writing series" size="30 MB"
level=debug ts=2026-05-07T15:16:10.769338743Z caller=checkpoint.go:384 msg="flushing series" totalSize="30 MB" series=6
level=debug ts=2026-05-07T15:16:14.111556387Z caller=checkpoint.go:372 msg="writing series" size="559 kB"
level=warn ts=2026-05-07T15:16:14.986234858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:17.48637491Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-07T15:16:17.486401676Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-07T15:16:19.986379964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:16:20.191455946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:16:20.861737768Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-07T15:16:23.952315287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="236 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:16:23.959396916Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:16:24.221108357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:16:24.236209027Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-07T15:16:24.986328784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:27.611076304Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=warn ts=2026-05-07T15:16:29.985594339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:16:30.137057595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:16:30.1450908Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:16:30.986688283Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-07T15:16:33.379860333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:16:34.361834686Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-07T15:16:34.652576156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:16:34.660558575Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:16:34.986166143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:37.736457614Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-07T15:16:39.98557889Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:16:39.985601005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:39.9856118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:16:39.985622073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:16:41.111760429Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=debug ts=2026-05-07T15:16:44.486431641Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-07T15:16:44.985669617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:47.860886339Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-07T15:16:49.986518457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:16:50.19391435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:16:51.236117705Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=debug ts=2026-05-07T15:16:54.611237852Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-07T15:16:54.985942495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:16:57.597334943Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:16:57.987220188Z caller=checkpoint.go:372 msg="writing series" size="891 kB"
level=debug ts=2026-05-07T15:16:57.987243476Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=12
level=debug ts=2026-05-07T15:16:58.560316185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:16:59.097780497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42114"
level=warn ts=2026-05-07T15:16:59.985509461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:01.361166793Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-07T15:17:04.735913713Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-07T15:17:04.98578266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:17:06.525074088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="259 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:17:06.534167651Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:17:08.111843482Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:17:09.966218926Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:17:09.966671278Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:17:09.986024231Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:17:09.986043669Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:17:09.986038227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:17:09.986052074Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:17:11.486161838Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-07T15:17:12.880321345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:17:12.889136791Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:17:14.86188569Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:17:14.985896803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:18.236314382Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=warn ts=2026-05-07T15:17:19.985612632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:17:20.196841513Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:17:21.611925526Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=warn ts=2026-05-07T15:17:24.985658949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:24.986781666Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-05-07T15:17:27.136483834Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:17:27.136550444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:17:27.145088118Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:17:28.361736621Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
ts=2026-05-07T15:17:29.09882451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:52174"
level=warn ts=2026-05-07T15:17:29.986004944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:17:30.199220507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:17:30.207645166Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:17:31.736329502Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-07T15:17:33.476779282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:17:34.986017053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:35.111335174Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=debug ts=2026-05-07T15:17:38.486342602Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=info ts=2026-05-07T15:17:39.006855013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:17:39.010984199Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:17:39.985613776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:17:39.98560883Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:17:39.985648641Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:17:39.985657619Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:17:41.861872483Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-07T15:17:41.861897102Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-07T15:17:44.986339672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:45.2364739Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=info ts=2026-05-07T15:17:46.90191817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:17:46.910762561Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:17:48.610952993Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-07T15:17:49.985539929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:17:50.199330626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:17:51.986540202Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-07T15:17:54.985637831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:17:55.360984668Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-07T15:17:58.736643984Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T15:17:59.797840747Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:17:59.986322441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:02.111746141Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-07T15:18:04.986234593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:18:05.020388273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43032"
level=debug ts=2026-05-07T15:18:05.486218397Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-05-07T15:18:08.005328077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:18:08.012204698Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:18:08.861682312Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-07T15:18:09.966190122Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:18:09.967724408Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:18:09.986378477Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:18:09.986401133Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:18:09.986409053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:18:09.986423657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:18:11.419313314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:18:11.426182579Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:18:11.688957699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:18:11.69553278Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:18:12.236170919Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-07T15:18:14.986318841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:15.611163648Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-07T15:18:15.611187181Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=debug ts=2026-05-07T15:18:18.985992832Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:18:19.985910932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:18:20.20091838Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:18:22.36099489Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-07T15:18:23.920534732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:18:24.080292557Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:18:24.986440439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:25.736844062Z caller=checkpoint.go:372 msg="writing series" size="584 kB"
ts=2026-05-07T15:18:26.803330513Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50116"
level=debug ts=2026-05-07T15:18:29.11187678Z caller=checkpoint.go:372 msg="writing series" size="448 kB"
level=warn ts=2026-05-07T15:18:29.985599166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:32.487093585Z caller=checkpoint.go:372 msg="writing series" size="790 kB"
level=debug ts=2026-05-07T15:18:32.487131914Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=warn ts=2026-05-07T15:18:34.985851016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:35.861405226Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-05-07T15:18:39.236671054Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-05-07T15:18:39.98596782Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:18:39.985995238Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:18:39.985985017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:18:39.986003552Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:18:40.050315478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:18:40.061808185Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:18:40.061832905Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:18:40.061863756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="34 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:18:40.068873825Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:18:42.611517379Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-07T15:18:44.98569619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:45.986909757Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-07T15:18:49.361405565Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-07T15:18:49.361429118Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-05-07T15:18:49.986104995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:18:50.203846127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:18:52.737676493Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:18:52.737698682Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=warn ts=2026-05-07T15:18:54.986378466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:18:55.107040482Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:18:55.107059955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:18:55.120515521Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:18:56.112452562Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-07T15:18:56.112482878Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=info ts=2026-05-07T15:18:56.510840158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="305 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:18:56.518628732Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:18:58.514076316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:18:58.518426753Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:18:59.486379374Z caller=checkpoint.go:372 msg="writing series" size="286 kB"
level=warn ts=2026-05-07T15:18:59.985494056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:02.86134765Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=warn ts=2026-05-07T15:19:04.985826788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:19:05.027848734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33600"
level=debug ts=2026-05-07T15:19:06.236874372Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=debug ts=2026-05-07T15:19:09.611646246Z caller=checkpoint.go:372 msg="writing series" size="476 kB"
level=debug ts=2026-05-07T15:19:09.611669782Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-07T15:19:09.967161083Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:19:09.968209327Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T15:19:09.986292381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:19:09.986296439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:19:09.986320342Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:19:09.98633041Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:19:10.373872861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:19:10.384469116Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:19:11.257096133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:19:11.265226029Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:19:12.595904506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:19:12.604702036Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:19:12.986945216Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=warn ts=2026-05-07T15:19:14.985945414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:16.36149018Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-07T15:19:19.741359816Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-07T15:19:19.741388238Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=3
level=warn ts=2026-05-07T15:19:19.985481655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:19:20.217424283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:19:21.154791773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:19:23.111212626Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-07T15:19:23.791001931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="37 MB" avg_comp="2.9 MB" total_uncomp="305 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:19:23.801853635Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:19:24.985646014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:25.366426375Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:19:25.368347423Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:19:26.487024618Z caller=checkpoint.go:372 msg="writing series" size="696 kB"
ts=2026-05-07T15:19:26.806896396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48572"
level=debug ts=2026-05-07T15:19:29.861869203Z caller=checkpoint.go:372 msg="writing series" size="436 kB"
level=debug ts=2026-05-07T15:19:29.861897478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-05-07T15:19:29.985620938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:33.236717263Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-07T15:19:34.985641805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:36.61150344Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T15:19:39.985433346Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:19:39.985460821Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:19:39.985468942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:19:39.985504717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:19:39.986517516Z caller=checkpoint.go:384 msg="flushing series" totalSize="83 kB" series=2
level=info ts=2026-05-07T15:19:39.989122482Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-07T15:19:39.991537134Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00576379s
level=warn ts=2026-05-07T15:19:44.986563137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:19:45.235900825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:19:45.244868162Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:19:49.985851373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:19:50.22084735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:19:51.860412371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:19:52.158395104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:19:52.163038537Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:19:54.74952777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:19:54.758011829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:19:54.986009503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:19:59.109694293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49838"
level=warn ts=2026-05-07T15:19:59.985966778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:20:04.193050291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:20:04.208761962Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:20:04.986304343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:20:09.96630184Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:20:09.967687049Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:20:09.986187364Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T15:20:09.986196426Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-07T15:20:09.986211Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:20:09.986220402Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:20:09.986239776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:20:09.986328273Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-07T15:20:09.986337231Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:20:09.986354402Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.186µs success=true
level=debug ts=2026-05-07T15:20:09.986692074Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-07T15:20:13.322127857Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-07T15:20:13.3221512Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=2
level=warn ts=2026-05-07T15:20:14.986515671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:15.570894105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:20:16.653971945Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=info ts=2026-05-07T15:20:18.46362678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="329 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:20:18.474861512Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:20:19.986038081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:19.991883816Z caller=checkpoint.go:372 msg="writing series" size="7.8 MB"
level=debug ts=2026-05-07T15:20:19.991908828Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.8 MB" series=2
ts=2026-05-07T15:20:20.224443042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:20:23.320031689Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-07T15:20:24.530406558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:20:24.986578013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:26.654007669Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T15:20:29.110949595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49988"
level=warn ts=2026-05-07T15:20:29.986052674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:29.987335115Z caller=checkpoint.go:372 msg="writing series" size="486 kB"
level=debug ts=2026-05-07T15:20:33.32034884Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-07T15:20:34.985905619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:20:35.038314204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:41948"
level=info ts=2026-05-07T15:20:35.117979938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:20:35.125989025Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:20:36.42186417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:20:36.426179413Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:20:36.653855586Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:20:39.289317511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:20:39.300091756Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:20:39.986283682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:20:39.986310035Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:20:39.986316889Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:20:39.986351011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:39.987484049Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-07T15:20:43.319947082Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-07T15:20:44.985758785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:46.653724297Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-05-07T15:20:49.986499043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:49.986671669Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-05-07T15:20:49.986686514Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
ts=2026-05-07T15:20:50.227398833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:20:51.022552367Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:20:53.320867695Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-07T15:20:54.562798156Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:20:54.986217991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:20:55.819787077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:20:55.82803258Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:20:56.653315769Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
ts=2026-05-07T15:20:56.81138808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51304"
ts=2026-05-07T15:20:59.112660867Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:32768"
level=warn ts=2026-05-07T15:20:59.985533974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:20:59.986910293Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=info ts=2026-05-07T15:21:03.254333566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="329 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:21:03.264517739Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:21:03.320607037Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:21:04.986313979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:06.653819875Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-07T15:21:09.966527903Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:21:09.967049942Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:21:09.985503789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:21:09.985526552Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:21:09.985535114Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:21:09.985550955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:10.054096785Z caller=checkpoint.go:372 msg="writing series" size="43 MB"
level=debug ts=2026-05-07T15:21:10.054129677Z caller=checkpoint.go:384 msg="flushing series" totalSize="43 MB" series=6
level=debug ts=2026-05-07T15:21:13.320980473Z caller=checkpoint.go:372 msg="writing series" size="524 kB"
level=warn ts=2026-05-07T15:21:14.986318448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:16.653555301Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-07T15:21:16.653582385Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=2
level=info ts=2026-05-07T15:21:19.441489448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:21:19.446491307Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:21:19.986312446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:19.987381347Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-07T15:21:20.230028073Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:21:23.085554043Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:21:23.320232461Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-07T15:21:24.455722046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:21:24.46408321Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:21:24.524150535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:21:24.530665412Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:21:24.986081063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:25.484711962Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:21:26.65405878Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-07T15:21:29.985472893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:29.986544699Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-07T15:21:33.32060278Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=warn ts=2026-05-07T15:21:34.986311886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:36.653283815Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-07T15:21:39.985725858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:21:39.985734414Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:21:39.985755321Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:21:39.985763003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:21:39.986862552Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-07T15:21:43.320809026Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-07T15:21:43.977039413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="329 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:21:43.991955039Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:21:44.985691155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:46.653634433Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-07T15:21:49.985468451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:49.98653927Z caller=checkpoint.go:372 msg="writing series" size="513 B"
ts=2026-05-07T15:21:50.232615162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:21:51.370448009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:21:51.384543685Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:21:51.384569158Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:21:51.384596357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:21:51.391832667Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:21:53.169805404Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:21:53.320206319Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-07T15:21:54.985718348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:56.653832413Z caller=checkpoint.go:372 msg="writing series" size="870 kB"
level=debug ts=2026-05-07T15:21:56.65386114Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=12
ts=2026-05-07T15:21:56.81618646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33982"
level=warn ts=2026-05-07T15:21:59.986472883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:21:59.987582219Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-07T15:22:03.176719357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:22:03.183853958Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:22:03.320192071Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=warn ts=2026-05-07T15:22:04.986169236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:22:05.055076597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49188"
level=info ts=2026-05-07T15:22:05.509008306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:22:05.516383479Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:22:05.539456699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:22:05.548567308Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:22:06.653703782Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:22:09.966189161Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:22:09.966559495Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:22:09.986377421Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:22:09.986389415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:09.986405234Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:22:09.986414045Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:22:09.987633831Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=debug ts=2026-05-07T15:22:13.320946884Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:22:14.986389319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:16.65394219Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-07T15:22:18.696554894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:22:18.70799665Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:22:18.708024091Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:22:18.708051629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:22:18.715166195Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:22:19.985952382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:19.98712142Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
ts=2026-05-07T15:22:20.235174301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:22:21.9755626Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:22:23.320514336Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=debug ts=2026-05-07T15:22:23.320544124Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-07T15:22:24.56640058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:22:24.98601304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:26.65385845Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T15:22:27.253740908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="353 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:22:27.262376679Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:22:28.047719673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:22:28.056318216Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:22:29.985542721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:29.986683801Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-07T15:22:33.320537483Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-07T15:22:34.6116028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:22:34.619903539Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:22:34.986476721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:36.654083474Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-07T15:22:39.986376265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:22:39.986377361Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:22:39.986401854Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:22:39.986409683Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:22:39.987583022Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-07T15:22:43.320681897Z caller=checkpoint.go:372 msg="writing series" size="987 kB"
level=debug ts=2026-05-07T15:22:43.320705539Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-07T15:22:44.985525862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:22:46.638175754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:22:46.646053475Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:22:46.654143043Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-07T15:22:49.98652185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:49.986675911Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
ts=2026-05-07T15:22:50.238517099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:22:53.00917121Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:22:53.320927374Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-07T15:22:54.985592909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:56.653547694Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=warn ts=2026-05-07T15:22:59.986339732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:22:59.987508875Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=debug ts=2026-05-07T15:23:03.320693633Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=info ts=2026-05-07T15:23:04.173456471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:23:04.182932456Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:23:04.985763603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:06.65421325Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:23:08.920641183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:23:08.922475464Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:23:09.966354408Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:23:09.966825708Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T15:23:09.986225894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:23:09.986231553Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:23:09.986272117Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:23:09.986285536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:23:09.987621728Z caller=checkpoint.go:372 msg="writing series" size="365 kB"
level=debug ts=2026-05-07T15:23:09.987634791Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-07T15:23:10.125947874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="376 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:23:10.133561546Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:23:11.806487662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:23:11.814118789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:23:13.32033137Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-07T15:23:14.985622287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:16.653743069Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-07T15:23:17.556311184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:23:17.562652042Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:23:19.98605868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:19.987198166Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-07T15:23:20.241971392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:23:20.389297951Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:23:23.321186696Z caller=checkpoint.go:372 msg="writing series" size="944 kB"
level=warn ts=2026-05-07T15:23:24.985866248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:26.654518996Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-07T15:23:26.654543899Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-05-07T15:23:29.986265283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:29.987602432Z caller=checkpoint.go:372 msg="writing series" size="888 kB"
level=info ts=2026-05-07T15:23:31.379673081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:23:31.387599702Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:23:31.387626681Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:23:31.387652083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:23:31.391247983Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:23:33.320702231Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-07T15:23:34.986234406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:36.653620913Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-07T15:23:36.653644518Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-07T15:23:39.986379296Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:23:39.986391977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:39.986409186Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:23:39.986417339Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:23:39.987666939Z caller=checkpoint.go:372 msg="writing series" size="360 kB"
level=debug ts=2026-05-07T15:23:43.320165968Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-07T15:23:44.985995704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:23:45.547319995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:23:45.554039454Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:23:46.654069993Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-07T15:23:49.601054759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:23:49.608986027Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:23:49.985939419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:49.987160044Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-05-07T15:23:50.244918155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:23:52.17603526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:23:52.319431231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="376 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:23:52.326174719Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:23:53.321087905Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-07T15:23:53.321110858Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=5
level=warn ts=2026-05-07T15:23:54.986531585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:56.654610977Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-07T15:23:56.654634063Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=1
ts=2026-05-07T15:23:56.821696059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37032"
level=warn ts=2026-05-07T15:23:59.985630338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:23:59.986836735Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-05-07T15:24:00.461913721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:24:03.320520321Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=info ts=2026-05-07T15:24:03.742703286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:24:03.751999251Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:24:04.98591666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:06.654194571Z caller=checkpoint.go:372 msg="writing series" size="606 kB"
level=info ts=2026-05-07T15:24:09.966225317Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:24:09.966231304Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:24:09.985927152Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:24:09.985950637Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:24:09.985957696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:24:09.985955422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:09.987168791Z caller=checkpoint.go:372 msg="writing series" size="388 kB"
level=debug ts=2026-05-07T15:24:09.987178434Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-07T15:24:10.909467142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:24:10.917291339Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:24:13.320384836Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=warn ts=2026-05-07T15:24:14.986181596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:16.654118608Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-07T15:24:18.77210653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:24:18.77729285Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:24:19.985862541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:19.989761447Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-07T15:24:19.989783727Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=3
ts=2026-05-07T15:24:20.247033472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:24:23.320327999Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-07T15:24:24.622680731Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:24:24.986309889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:26.654455101Z caller=checkpoint.go:372 msg="writing series" size="765 kB"
level=warn ts=2026-05-07T15:24:29.985627002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:29.987006993Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T15:24:29.987023531Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-05-07T15:24:33.321014011Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=warn ts=2026-05-07T15:24:34.985884258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:24:35.070290564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51998"
level=info ts=2026-05-07T15:24:36.368721661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="376 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:24:36.377571824Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:24:36.377595411Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:24:36.377628423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="376 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:24:36.385070057Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:24:36.653691886Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T15:24:39.985995577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:24:39.986020873Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:24:39.986029043Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:24:39.986020833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:24:39.987095923Z caller=checkpoint.go:384 msg="flushing series" totalSize="134 kB" series=2
level=info ts=2026-05-07T15:24:39.991394549Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-07T15:24:39.994390471Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007903183s
level=debug ts=2026-05-07T15:24:43.41692849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:24:44.550912465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:24:44.559192156Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:24:44.986169049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:24:47.940328213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="376 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:24:47.947212869Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:24:49.985648708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:24:50.248888536Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:24:52.283323092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:24:52.290489729Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:24:54.912352069Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:24:54.986524743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:24:59.985490672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:25:04.98570881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:25:09.252314398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:25:09.264080489Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:25:09.966280232Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:25:09.967543163Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T15:25:09.985538279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:25:09.985548665Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:25:09.985564968Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:25:09.985585504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:25:09.985593419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:25:09.985719157Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-07T15:25:09.985728705Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:25:09.985745628Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.207µs success=true
level=debug ts=2026-05-07T15:25:09.985992022Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-07T15:25:13.322829154Z caller=checkpoint.go:372 msg="writing series" size="4.5 MB"
level=debug ts=2026-05-07T15:25:13.322852755Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.7 MB" series=2
level=warn ts=2026-05-07T15:25:14.985957433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:16.652837475Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=info ts=2026-05-07T15:25:16.951735228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:25:16.955939265Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:25:19.985475747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:19.991756006Z caller=checkpoint.go:372 msg="writing series" size="8.5 MB"
level=debug ts=2026-05-07T15:25:19.991775444Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.5 MB" series=2
ts=2026-05-07T15:25:20.250808242Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:25:23.320349379Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-07T15:25:24.986052396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:25:25.355549792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="376 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:25:25.362853952Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:25:26.652683952Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T15:25:26.826764788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34284"
level=warn ts=2026-05-07T15:25:29.985760726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:29.987069786Z caller=checkpoint.go:372 msg="writing series" size="486 kB"
level=debug ts=2026-05-07T15:25:30.16141286Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:25:33.32022696Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-07T15:25:34.986557531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:36.653521312Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:25:39.985422305Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:25:39.985447655Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:25:39.985455848Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:25:39.985471693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:39.986543498Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-07T15:25:43.319598101Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-07T15:25:43.589147498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:25:43.596537526Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=error ts=2026-05-07T15:25:43.596562863Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:25:43.59658749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:25:43.606227282Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:25:43.743446334Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:25:44.985891688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:46.653444078Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-07T15:25:47.377762207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:25:47.384325298Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:25:49.985763369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:49.986876375Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
ts=2026-05-07T15:25:50.25173742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T15:25:51.816464927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:25:51.826413055Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:25:53.31980052Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:25:54.986358769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:56.652691903Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-07T15:25:58.489573253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:25:58.497825608Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
ts=2026-05-07T15:25:59.129617959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42948"
level=warn ts=2026-05-07T15:25:59.985478323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:25:59.98674057Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-07T15:25:59.986752433Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=12
level=debug ts=2026-05-07T15:26:03.31981297Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:26:04.98619047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:05.72572183Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:26:06.415524458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:26:06.420761872Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:26:06.653506822Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-07T15:26:09.967017436Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:26:09.967534451Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T15:26:09.985931566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:26:09.985962539Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:26:09.985974524Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:26:09.985984009Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:26:10.040072565Z caller=checkpoint.go:372 msg="writing series" size="48 MB"
level=debug ts=2026-05-07T15:26:10.040102305Z caller=checkpoint.go:384 msg="flushing series" totalSize="49 MB" series=3
level=debug ts=2026-05-07T15:26:13.321037122Z caller=checkpoint.go:372 msg="writing series" size="654 kB"
level=warn ts=2026-05-07T15:26:14.986447744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:16.653608182Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-07T15:26:16.653635177Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-07T15:26:18.173248149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="376 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:26:18.187977089Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:26:18.988659194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:26:18.996506261Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:26:19.985851132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:19.986900471Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-07T15:26:20.252744574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:26:23.320390224Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-07T15:26:24.986142045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:26.653121417Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-07T15:26:29.986309767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:29.98635502Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-07T15:26:33.320141099Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-07T15:26:34.985917044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:26:35.08206288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:60964"
level=info ts=2026-05-07T15:26:36.19268636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:26:36.200894839Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:26:36.653133498Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-05-07T15:26:37.031443285Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:26:39.310043836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:26:39.31690522Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:26:39.986378407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:26:39.986379823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:39.986401687Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:26:39.98641058Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:26:39.986419215Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-07T15:26:40.902981896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:26:43.319950133Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-07T15:26:44.986400887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:46.653449898Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-07T15:26:49.98555626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:49.986658822Z caller=checkpoint.go:372 msg="writing series" size="513 B"
ts=2026-05-07T15:26:50.253829398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:26:53.320044865Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-07T15:26:53.834618745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:26:53.847360175Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:26:54.985873631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:56.653825407Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T15:26:56.653851692Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=12
level=warn ts=2026-05-07T15:26:59.985639854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:26:59.986763071Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-07T15:27:02.714638435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:27:03.320432789Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=warn ts=2026-05-07T15:27:04.985903915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:27:05.085106649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45416"
level=debug ts=2026-05-07T15:27:06.653420935Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:27:09.966548525Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:27:09.967280792Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:27:09.985485653Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:27:09.985490585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:09.985510323Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:27:09.985520266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:27:09.986615148Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-07T15:27:10.555331404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:27:10.563924388Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:27:10.895845639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:27:10.905468574Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:27:13.319584919Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:27:14.986090336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:16.653396107Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-07T15:27:19.986000077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:19.986073978Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
ts=2026-05-07T15:27:20.25467184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:27:21.418458123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:27:21.428968733Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:27:23.320208598Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
level=warn ts=2026-05-07T15:27:24.986122419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:26.653665073Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-07T15:27:29.986083019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:29.986102345Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-07T15:27:30.37190365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:27:30.37840765Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:27:30.797656382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:27:30.799500871Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:27:33.319985791Z caller=checkpoint.go:372 msg="writing series" size="8.6 kB"
level=warn ts=2026-05-07T15:27:34.985875811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:36.628136132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:27:36.65328288Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-07T15:27:36.653305926Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-05-07T15:27:39.985562267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:27:39.98559023Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:27:39.98559831Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:27:39.98558922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:39.986759926Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-07T15:27:43.320436787Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T15:27:43.320459842Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-07T15:27:44.986461479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:46.653712538Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-07T15:27:49.835468249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:27:49.840306466Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:27:49.986025222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:49.986093497Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-07T15:27:50.256419966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:27:53.319928943Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-07T15:27:54.98567596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:27:56.250218092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:27:56.259528781Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:27:56.653644765Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
ts=2026-05-07T15:27:56.835867533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34278"
level=warn ts=2026-05-07T15:27:59.986390651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:27:59.986541128Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=debug ts=2026-05-07T15:28:03.319999273Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-07T15:28:04.986328859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:06.653330085Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:28:07.306330652Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:28:07.306396132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:28:07.314863119Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:28:09.966215553Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:28:09.966498046Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:28:09.985910229Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:28:09.985911575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:09.985932919Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:28:09.985940345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:28:09.986066759Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=debug ts=2026-05-07T15:28:09.986077826Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-07T15:28:10.850492015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:28:10.858126787Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:28:11.091965775Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:28:13.320272387Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-07T15:28:14.985698106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:16.652692598Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-07T15:28:19.553231963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:28:19.559470799Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:28:19.559492318Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:28:19.559518611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:28:19.566044676Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:28:19.986050056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:19.986078398Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-07T15:28:20.258146523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:28:21.035494152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:28:21.042492446Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:28:23.320673623Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=warn ts=2026-05-07T15:28:24.986144192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:28:25.029303237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:28:25.035343652Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:28:26.653903635Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-07T15:28:26.653927901Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
ts=2026-05-07T15:28:29.141205392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38250"
level=warn ts=2026-05-07T15:28:29.986369652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:29.987299816Z caller=checkpoint.go:372 msg="writing series" size="900 kB"
level=debug ts=2026-05-07T15:28:33.320281012Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-07T15:28:33.55899354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:28:33.566219055Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:28:34.986424012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:28:35.095304255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55650"
level=debug ts=2026-05-07T15:28:36.653861032Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-07T15:28:36.653886286Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=debug ts=2026-05-07T15:28:39.435576651Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:28:39.985909153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:28:39.985913907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:39.985932874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:28:39.985942413Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:28:39.986429216Z caller=checkpoint.go:372 msg="writing series" size="456 kB"
level=info ts=2026-05-07T15:28:42.748172762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:28:42.752772014Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:28:43.319580201Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-07T15:28:44.985676768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:46.653180847Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-07T15:28:47.058507829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:28:47.066951652Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:28:49.986038607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:49.986172445Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-07T15:28:49.986188687Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-07T15:28:50.259664512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:28:53.322243435Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-07T15:28:53.322290098Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
level=warn ts=2026-05-07T15:28:54.985737691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:28:55.586503677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:28:55.595084801Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:28:56.654691331Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-07T15:28:56.654714444Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=1
level=warn ts=2026-05-07T15:28:59.986476964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:28:59.986518954Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=debug ts=2026-05-07T15:29:02.402687824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:29:03.320113779Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=info ts=2026-05-07T15:29:04.922898669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:29:04.930433089Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:29:04.986099605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:06.653597711Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-07T15:29:08.904875482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:29:08.913241978Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:29:08.9132811Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:29:08.913306146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:29:08.919177142Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:29:09.585224659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:29:09.591321889Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:29:09.966186789Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:29:09.966654837Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:29:09.985817789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:29:09.985842117Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:29:09.985851245Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:29:09.985866787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:09.987197416Z caller=checkpoint.go:372 msg="writing series" size="588 kB"
level=debug ts=2026-05-07T15:29:09.987214123Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-07T15:29:13.320056458Z caller=checkpoint.go:372 msg="writing series" size="423 kB"
level=debug ts=2026-05-07T15:29:14.757678501Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:29:14.986236121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:16.653272148Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-07T15:29:19.986111545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:19.988309368Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-07T15:29:19.988325224Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=3
ts=2026-05-07T15:29:20.261468244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:29:23.320265839Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-07T15:29:24.986493141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:26.653537305Z caller=checkpoint.go:372 msg="writing series" size="802 kB"
ts=2026-05-07T15:29:26.841020828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48974"
level=info ts=2026-05-07T15:29:27.260669578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:29:27.269921434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:29:29.985756883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:29.987210665Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T15:29:29.987226249Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-05-07T15:29:33.320312726Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T15:29:33.361438564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:29:33.367546446Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:29:34.985536779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:29:36.086837476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:29:36.094250156Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:29:36.652685343Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T15:29:38.503222437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:29:38.510640472Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:29:39.397387422Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:29:39.985957082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:29:39.985986911Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-07T15:29:39.985986459Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T15:29:39.986016686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:29:39.986025263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:29:39.990518576Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-07T15:29:39.993144678Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007253665s
level=warn ts=2026-05-07T15:29:44.985873009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:29:49.986083445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:29:50.263767304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:29:51.590235084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:29:51.600028579Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:29:52.687005885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:29:52.693270472Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:29:54.986202879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:29:56.78414451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:29:56.842458669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34614"
level=warn ts=2026-05-07T15:29:59.985615068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:30:04.985665898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:30:06.705130432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:30:06.713797479Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:30:09.966182078Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:30:09.966502427Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=debug ts=2026-05-07T15:30:09.967865494Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T15:30:09.967908485Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975741 n=1
level=debug ts=2026-05-07T15:30:09.967950161Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975741 n=1
level=info ts=2026-05-07T15:30:09.985831216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:30:09.985850706Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:30:09.985858254Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:30:09.985853638Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T15:30:09.985879082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:30:09.986096138Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-07T15:30:09.986105709Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:30:09.986123979Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.526µs success=true
level=debug ts=2026-05-07T15:30:09.986399841Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-07T15:30:13.328269503Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T15:30:13.328297054Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=debug ts=2026-05-07T15:30:13.497183445Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:30:14.98561546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:16.653036845Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=info ts=2026-05-07T15:30:19.313983255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:30:19.324167036Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:30:19.954876101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:30:19.959037014Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:30:19.959057895Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:30:19.959078911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:30:19.962425965Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:30:19.986223535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:19.994512195Z caller=checkpoint.go:372 msg="writing series" size="8.8 MB"
level=debug ts=2026-05-07T15:30:19.994538288Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.8 MB" series=2
ts=2026-05-07T15:30:20.266361678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:30:23.320389335Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-07T15:30:24.986197397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:26.653592151Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-07T15:30:29.986316101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:29.986740763Z caller=checkpoint.go:372 msg="writing series" size="486 kB"
level=info ts=2026-05-07T15:30:31.759876841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:30:31.769999415Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:30:33.320242245Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-07T15:30:33.690313161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:30:33.694744945Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:30:34.986215987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:36.653591352Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:30:37.446803504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="6.2 MB" avg_comp="2.1 MB" total_uncomp="53 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:30:37.455689329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:30:39.985600894Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:30:39.985628637Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:30:39.985640519Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:30:39.985641597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:39.986782926Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-07T15:30:40.585813901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:30:40.592173248Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:30:43.319807392Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-07T15:30:44.985829935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:46.654463136Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:30:46.654491294Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=8
level=debug ts=2026-05-07T15:30:48.480295901Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:30:49.985805578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:49.98696772Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
ts=2026-05-07T15:30:50.268346889Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:30:50.803638643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:30:53.320267101Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:30:54.985961592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:56.653683437Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-07T15:30:59.985973535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:30:59.987340598Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=info ts=2026-05-07T15:31:00.107281695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:31:00.11568939Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:31:03.320279483Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-07T15:31:04.103744829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:31:04.107899381Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:31:04.985817154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:06.653582271Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-07T15:31:09.966313215Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:31:09.966838143Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:31:09.986416756Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:31:09.986436613Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:31:09.986444406Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:31:09.986485427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:10.05373768Z caller=checkpoint.go:372 msg="writing series" size="50 MB"
level=debug ts=2026-05-07T15:31:10.053776661Z caller=checkpoint.go:384 msg="flushing series" totalSize="51 MB" series=7
level=debug ts=2026-05-07T15:31:13.321073009Z caller=checkpoint.go:372 msg="writing series" size="526 kB"
level=info ts=2026-05-07T15:31:14.102548491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:31:14.110418397Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:31:14.985547056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:31:15.314553836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:31:15.323118593Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:31:15.323141555Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:31:15.323176161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:31:15.329629036Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:31:16.653407186Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-07T15:31:16.653434996Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-07T15:31:19.986125262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:19.986330107Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-07T15:31:20.269396727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:31:22.046643814Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:31:23.32032998Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-07T15:31:24.190946199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:31:24.197281683Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:31:24.986537719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:26.6535465Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:31:29.985819516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:29.986928514Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-07T15:31:32.694742662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:31:32.702044019Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:31:33.319899131Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-07T15:31:34.073388036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:31:34.080539937Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:31:34.985613612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:36.653376442Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-07T15:31:39.985944047Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:31:39.985972587Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:31:39.985983175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:31:39.985997823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:39.987076596Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-07T15:31:43.320516883Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-07T15:31:44.332939483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:31:44.337147436Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:31:44.985864903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:46.653408996Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-07T15:31:47.086751534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:31:47.093270058Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:31:49.986410454Z caller=checkpoint.go:372 msg="writing series" size="513 B"
level=warn ts=2026-05-07T15:31:49.986433284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:31:50.271412839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:31:51.712002268Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:31:51.998720054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:31:52.000434603Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T15:31:53.320154091Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-07T15:31:54.986327785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:56.654950535Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T15:31:56.654972834Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=12
level=info ts=2026-05-07T15:31:56.745031953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:31:56.752851785Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=error ts=2026-05-07T15:31:56.752873875Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:31:56.752899254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:31:56.759952991Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
ts=2026-05-07T15:31:56.849353293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:56952"
level=warn ts=2026-05-07T15:31:59.985658755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:31:59.986797565Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-07T15:32:03.320630368Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-07T15:32:03.364806455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:32:03.373807042Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:32:04.986158077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:06.653288814Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:32:09.350857462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:32:09.357170978Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:32:09.966380966Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:32:09.966742614Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:32:09.985918392Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:32:09.985938224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:32:09.985947102Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:32:09.985999184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:09.987144637Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-07T15:32:13.320267884Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:32:14.985532127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:32:15.9152762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:32:15.925032679Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:32:16.653157401Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-07T15:32:19.986470744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:19.986571074Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-07T15:32:20.271246578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
ts=2026-05-07T15:32:20.274402327Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=error ts=2026-05-07T15:32:20.279574928Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:32:23.320041534Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=warn ts=2026-05-07T15:32:24.985840456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:26.653674702Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-07T15:32:26.850805291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49756"
level=debug ts=2026-05-07T15:32:27.488976976Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:32:29.155796498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45628"
level=debug ts=2026-05-07T15:32:29.986374239Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-07T15:32:29.986392458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:33.320120356Z caller=checkpoint.go:372 msg="writing series" size="9.8 kB"
level=info ts=2026-05-07T15:32:34.470293051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:32:34.475543014Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:32:34.537613804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:32:34.546273216Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:32:34.985861852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:35.564837092Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:32:35.955232845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:32:35.962905719Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:32:36.653886371Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=debug ts=2026-05-07T15:32:36.653911932Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-05-07T15:32:39.985796206Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:32:39.985827645Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:32:39.985835746Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:32:39.985837085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:39.986965855Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=debug ts=2026-05-07T15:32:43.321129448Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T15:32:43.321153937Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-07T15:32:44.98577654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:46.653681188Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-07T15:32:49.985556114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:49.986728801Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
ts=2026-05-07T15:32:50.277030222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:32:53.320121983Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-05-07T15:32:53.731400319Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:32:54.627023711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:32:54.636447196Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:32:54.986207352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:56.653660057Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
ts=2026-05-07T15:32:59.157097246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58424"
level=warn ts=2026-05-07T15:32:59.986326943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:32:59.986489471Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-05-07T15:33:01.241932323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:33:01.250127156Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:33:03.320731441Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T15:33:04.985946425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:06.653953958Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-07T15:33:09.350719543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:33:09.359191474Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:33:09.966337488Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:33:09.966635605Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:33:09.986234927Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:33:09.986268931Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:33:09.986277908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:33:09.98628494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:09.986471727Z caller=checkpoint.go:372 msg="writing series" size="386 kB"
level=debug ts=2026-05-07T15:33:09.986484676Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-07T15:33:11.468328764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:33:13.320519814Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-07T15:33:14.986303103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:33:16.409300567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:33:16.416451939Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:33:16.653041489Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-07T15:33:18.531165841Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:33:19.986320421Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-07T15:33:19.986326706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:33:20.279701259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:33:20.788123233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:33:20.791872862Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:33:22.545950056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:33:22.552401398Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:33:23.320627797Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=warn ts=2026-05-07T15:33:24.986222673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:26.653559826Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-07T15:33:26.653583773Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
ts=2026-05-07T15:33:26.855516278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:52254"
ts=2026-05-07T15:33:29.158684497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38292"
level=warn ts=2026-05-07T15:33:29.986097083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:29.987832425Z caller=checkpoint.go:372 msg="writing series" size="981 kB"
level=debug ts=2026-05-07T15:33:33.319639215Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-07T15:33:34.986417583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:36.654345172Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-07T15:33:36.654378696Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-07T15:33:38.366020565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:33:38.372768049Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:33:39.985733463Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:33:39.985758664Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:33:39.985766312Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:33:39.985754961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:39.987127853Z caller=checkpoint.go:372 msg="writing series" size="324 kB"
level=debug ts=2026-05-07T15:33:43.320164856Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-07T15:33:44.985876661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:46.654124843Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=warn ts=2026-05-07T15:33:49.985912088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:49.987114924Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-07T15:33:50.101309436Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:33:50.282147391Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:33:51.659220544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:33:51.669055917Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:33:52.01157305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:33:52.019098237Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:33:53.320716168Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-07T15:33:53.320741608Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=5
level=warn ts=2026-05-07T15:33:54.985606729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:56.655659784Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-07T15:33:56.655682636Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=1
level=info ts=2026-05-07T15:33:57.14049446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:33:57.149004486Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
ts=2026-05-07T15:33:59.159788133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38788"
level=warn ts=2026-05-07T15:33:59.986109188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:33:59.987359694Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=debug ts=2026-05-07T15:34:02.297646506Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:34:03.320639259Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=warn ts=2026-05-07T15:34:04.986442204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:34:05.129302613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51234"
level=debug ts=2026-05-07T15:34:06.653494783Z caller=checkpoint.go:372 msg="writing series" size="462 kB"
level=info ts=2026-05-07T15:34:06.782961324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:34:06.788710018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:34:09.966193859Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:34:09.966657162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:34:09.985935073Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:34:09.985953422Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:34:09.98594448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:34:09.98596072Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:34:09.98717986Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=debug ts=2026-05-07T15:34:09.987192536Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-07T15:34:11.806299943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:34:11.814979852Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:34:13.32044691Z caller=checkpoint.go:372 msg="writing series" size="307 kB"
level=warn ts=2026-05-07T15:34:14.985976506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:15.598166408Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:34:16.653060816Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-07T15:34:19.986321173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:19.990308697Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-07T15:34:19.990325124Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=3
ts=2026-05-07T15:34:20.283312866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:34:23.320501755Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-07T15:34:24.986466265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:26.654160237Z caller=checkpoint.go:372 msg="writing series" size="818 kB"
level=info ts=2026-05-07T15:34:26.734145786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:34:26.744431098Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:34:26.744453342Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:34:26.744473845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:34:26.750382266Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T15:34:29.161177962Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37906"
level=warn ts=2026-05-07T15:34:29.986387402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:29.986833558Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T15:34:29.986848365Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-05-07T15:34:33.32060341Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T15:34:34.716508415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:34:34.723518054Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:34:34.986306506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:34:35.131956948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:42822"
level=info ts=2026-05-07T15:34:35.901214261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:34:35.907573097Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:34:36.653523347Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T15:34:39.98596306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:34:39.985989196Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:34:39.98598172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:34:39.985997747Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:34:39.987072576Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T15:34:39.997221234Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-07T15:34:39.998799877Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012570453s
level=info ts=2026-05-07T15:34:40.929896601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:34:40.9370191Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:34:44.986426482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:34:47.07397413Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:34:49.986395014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:34:50.284545462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:34:52.410025425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:34:52.419971103Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:34:53.639309892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:34:53.64594507Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:34:53.645967205Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:34:53.64598956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:34:53.652186248Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:34:54.986003461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:34:56.861865843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55142"
level=debug ts=2026-05-07T15:34:57.623750621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:34:59.363306558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:34:59.366959631Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:34:59.986391717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:35:04.66921458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:35:04.676434581Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:35:04.9863596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:35:09.966789133Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:35:09.967311114Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:35:09.985806561Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:35:09.98581688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:35:09.985820819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:09.985834188Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:35:09.98584254Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:35:09.985942799Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-07T15:35:09.985951957Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:35:09.985968679Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.629µs success=true
level=debug ts=2026-05-07T15:35:09.986247933Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-07T15:35:13.325863231Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T15:35:13.325884868Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=warn ts=2026-05-07T15:35:14.986371878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:16.255745051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:35:16.280862598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:35:16.28832389Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:35:16.653307666Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-07T15:35:19.986289946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:19.993982609Z caller=checkpoint.go:372 msg="writing series" size="9.8 MB"
level=debug ts=2026-05-07T15:35:19.994000296Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.8 MB" series=2
ts=2026-05-07T15:35:20.285339424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T15:35:20.75269306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:35:20.760299264Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:35:20.760320897Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:35:20.760347875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:35:20.767587998Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:35:22.223574317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:35:22.231998263Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:35:23.320243946Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-07T15:35:24.985634913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:26.653831271Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T15:35:26.863521359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37358"
level=info ts=2026-05-07T15:35:28.853629368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:35:28.861886518Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:35:29.985860694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:29.987148253Z caller=checkpoint.go:372 msg="writing series" size="486 kB"
level=info ts=2026-05-07T15:35:32.898880305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:35:32.906811497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:35:33.319926672Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:35:34.985539202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:36.653833844Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-07T15:35:38.547001056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:35:38.548772725Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:35:39.985501559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:35:39.985529609Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:35:39.98554058Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:35:39.985578602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:39.986705125Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=debug ts=2026-05-07T15:35:43.320022381Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-07T15:35:44.986304574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:46.653720821Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:35:46.65374237Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=8
level=info ts=2026-05-07T15:35:47.794725262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:35:47.800781578Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:35:48.933767989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:35:49.986373024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:49.986485696Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
ts=2026-05-07T15:35:50.286581361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T15:35:50.792880453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:35:50.800340495Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:35:53.320298826Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:35:54.986214548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:56.653324142Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:35:59.986384052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:35:59.986774355Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-07T15:36:03.319885558Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-07T15:36:04.986178506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:36:05.140474879Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:53704"
level=info ts=2026-05-07T15:36:06.05747943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:36:06.065558023Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:36:06.653280598Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-07T15:36:09.966186553Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:36:09.967666063Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:36:09.985487012Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:36:09.985505001Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:36:09.985493354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:36:09.98551257Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:36:10.049902822Z caller=checkpoint.go:372 msg="writing series" size="50 MB"
level=debug ts=2026-05-07T15:36:10.049927941Z caller=checkpoint.go:384 msg="flushing series" totalSize="51 MB" series=7
level=info ts=2026-05-07T15:36:11.595237624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:36:11.60566375Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:36:12.255743224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:36:12.262298187Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:36:13.32073158Z caller=checkpoint.go:372 msg="writing series" size="624 kB"
level=warn ts=2026-05-07T15:36:14.986010651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:15.03922064Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:36:15.754892663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:36:15.76121421Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:36:16.65427843Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-07T15:36:16.654304605Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-07T15:36:19.98610413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:19.986205844Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-07T15:36:20.288340054Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:36:23.320206579Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-05-07T15:36:24.986281956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:26.653466399Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-07T15:36:29.986096199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:29.986172461Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-07T15:36:33.320247485Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-07T15:36:34.986237968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:35.277541095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:36:36.653801844Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-07T15:36:36.684022394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:36:36.692478343Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:36:39.986199196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:36:39.986227275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:36:39.986233434Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-07T15:36:39.98624906Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:36:39.98627904Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:36:40.163950571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:36:40.168284218Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:36:40.168307443Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:36:40.168330979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:36:40.171906545Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:36:43.320584417Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-07T15:36:43.59959991Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:36:44.985632841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:46.653271479Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-07T15:36:49.985839145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:49.986946115Z caller=checkpoint.go:372 msg="writing series" size="642 B"
ts=2026-05-07T15:36:50.290065673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:36:50.53149699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:36:50.536558402Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:36:53.320549139Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-07T15:36:54.985865728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:36:56.41000556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:36:56.418049591Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:36:56.654846733Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T15:36:56.654869152Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=12
level=info ts=2026-05-07T15:36:57.664964245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:36:57.671139564Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T15:36:59.171724736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33166"
level=warn ts=2026-05-07T15:36:59.986125666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:36:59.986186536Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-07T15:37:00.569062381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:37:00.576375493Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:37:02.538110961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:37:02.545738628Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:37:03.320284027Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-07T15:37:04.985963744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:06.653331138Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-07T15:37:09.15103348Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:37:09.966220233Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:37:09.967050607Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:37:09.985640656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:37:09.985640553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:09.985663136Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:37:09.985670724Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:37:09.986951267Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-07T15:37:13.319921882Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:37:14.986460554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:16.653031177Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-07T15:37:19.986325814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:19.986528177Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-07T15:37:20.27293263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:37:20.277247521Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:37:20.292425514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:37:23.320729695Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-07T15:37:23.762798858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:37:23.770520018Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:37:24.986518828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:26.653439984Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-07T15:37:29.203281845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:37:29.985901165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:29.986987562Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-07T15:37:33.319949147Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-07T15:37:34.985617533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:36.653483918Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-07T15:37:36.653507211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=info ts=2026-05-07T15:37:39.985563175Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:37:39.985564493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:39.98558802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:37:39.98559589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:37:39.986798476Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=info ts=2026-05-07T15:37:40.84960248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:37:40.859896854Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:37:41.68235067Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:37:41.682413177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="828 kB" avg_comp="828 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:37:41.687090809Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:37:43.320156438Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T15:37:43.320180207Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-07T15:37:44.985485127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:46.653446508Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-07T15:37:47.837462885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:37:47.844577839Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:37:48.377207412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:37:48.382564268Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:37:49.985925294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:49.987078831Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
ts=2026-05-07T15:37:50.294812631Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:37:53.32023902Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=warn ts=2026-05-07T15:37:54.98640919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:37:55.055519456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:37:55.063487944Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:37:55.063512861Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:37:55.063557937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:37:55.071244491Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:37:56.653434976Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-07T15:37:59.626432213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:37:59.631285849Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:37:59.985499355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:37:59.986758095Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=debug ts=2026-05-07T15:38:03.319959088Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T15:38:04.985480224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:38:05.152361279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47682"
level=info ts=2026-05-07T15:38:05.395246777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:38:05.404387993Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:38:06.653413392Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-07T15:38:09.671651519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:38:09.680014374Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=error ts=2026-05-07T15:38:09.680036841Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:38:09.680063654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:38:09.688117628Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:38:09.966200714Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:38:09.967347466Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T15:38:09.986300325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:38:09.986306376Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:38:09.986331605Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:38:09.986341439Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:38:09.986546919Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=debug ts=2026-05-07T15:38:11.507193937Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:38:13.320423373Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-05-07T15:38:13.320446706Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-07T15:38:13.404709824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:38:13.410003754Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:38:14.986180355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:16.653668071Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:38:19.985537099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:19.986639389Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-07T15:38:20.296668047Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:38:22.524837643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:38:22.530178114Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:38:23.321111145Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=info ts=2026-05-07T15:38:23.86121625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:38:23.86905772Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:38:24.985512475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:26.654084508Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-07T15:38:26.654105499Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-07T15:38:28.712370403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:38:28.719723214Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:38:29.985538575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:29.987401325Z caller=checkpoint.go:372 msg="writing series" size="985 kB"
level=info ts=2026-05-07T15:38:30.050816038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:38:30.057369969Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:38:30.137393886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:38:30.143590805Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:38:30.259192142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:38:33.31956756Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-07T15:38:34.986429496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:38:35.15509798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:42756"
level=debug ts=2026-05-07T15:38:36.653897011Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-07T15:38:36.653918957Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-07T15:38:39.98587579Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:38:39.985887226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:39.985902082Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:38:39.985911977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:38:39.987213439Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=info ts=2026-05-07T15:38:40.331298526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:38:40.342404037Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:38:43.319998492Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-07T15:38:43.736208059Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:38:44.986440364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:46.653425414Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=warn ts=2026-05-07T15:38:49.985670128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:49.986852581Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-05-07T15:38:50.299109723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:38:53.323514835Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-07T15:38:53.323538606Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=5
level=warn ts=2026-05-07T15:38:54.985932849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:56.654911194Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-07T15:38:56.654933412Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
ts=2026-05-07T15:38:56.874674168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35228"
level=info ts=2026-05-07T15:38:58.56717957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:38:58.575616058Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
ts=2026-05-07T15:38:59.182684096Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45236"
level=warn ts=2026-05-07T15:38:59.985999258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:38:59.987153817Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-07T15:39:03.320504501Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=warn ts=2026-05-07T15:39:04.985992321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:39:05.499840017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:39:05.504606859Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:39:06.653965075Z caller=checkpoint.go:372 msg="writing series" size="495 kB"
level=info ts=2026-05-07T15:39:08.058652999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:39:08.063008548Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:39:09.918399962Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:39:09.966279439Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:39:09.96633463Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:39:09.986096595Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:39:09.986106828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:09.986120049Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:39:09.986127605Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:39:09.986295705Z caller=checkpoint.go:372 msg="writing series" size="400 kB"
level=debug ts=2026-05-07T15:39:13.320659577Z caller=checkpoint.go:372 msg="writing series" size="466 kB"
level=debug ts=2026-05-07T15:39:13.320682439Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-05-07T15:39:14.985725902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:39:15.40520186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:39:15.413283303Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:39:15.991973953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:39:15.997710118Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:39:16.653451023Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-07T15:39:19.123476682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:39:19.129928957Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:39:19.986162511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:19.989492459Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-07T15:39:19.989508413Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=2
ts=2026-05-07T15:39:20.30019553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:39:23.31964835Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:39:24.986172773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:25.708950981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:39:26.654046049Z caller=checkpoint.go:372 msg="writing series" size="854 kB"
ts=2026-05-07T15:39:26.876537436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:32982"
ts=2026-05-07T15:39:29.184028144Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55400"
level=warn ts=2026-05-07T15:39:29.986117905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:29.986621833Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T15:39:29.98663972Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-05-07T15:39:33.320340863Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T15:39:34.985882905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:36.653507338Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T15:39:39.234638721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:39:39.243427085Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:39:39.858163829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:39:39.865469463Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:39:39.985439456Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:39:39.985462925Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:39:39.985470816Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:39:39.985468449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:39:39.986557968Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T15:39:39.98922657Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-07T15:39:39.990372026Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004259746s
level=debug ts=2026-05-07T15:39:42.093609686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:39:44.985721436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:39:49.985625474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:39:50.301440197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=warn ts=2026-05-07T15:39:54.985705848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:39:56.877590253Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33076"
ts=2026-05-07T15:39:59.185415133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:41714"
level=warn ts=2026-05-07T15:39:59.98641388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:01.672703793Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:40:01.674676161Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:40:02.885305485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:40:02.892544092Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:40:03.565364642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:40:03.569345775Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:40:04.985530325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:40:06.06788746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:40:06.072786097Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:40:08.641941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:40:08.647652688Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:40:09.082932325Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:40:09.966201103Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:40:09.966415082Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:40:09.985397951Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:40:09.985423223Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:40:09.985442849Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:40:09.985451121Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:40:09.98547334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:40:09.985554682Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-07T15:40:09.985562278Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:40:09.985581506Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.576µs success=true
level=debug ts=2026-05-07T15:40:09.985857282Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-07T15:40:13.325591932Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T15:40:13.325615104Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=warn ts=2026-05-07T15:40:14.985612623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:16.652768853Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=info ts=2026-05-07T15:40:18.493794725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:40:18.502570479Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:40:19.985662088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:19.998306182Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-07T15:40:19.998337049Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
ts=2026-05-07T15:40:20.303324033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T15:40:22.0770211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:40:22.085144175Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:40:23.319626797Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-07T15:40:24.985945494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:26.653316351Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T15:40:26.879345914Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36548"
level=warn ts=2026-05-07T15:40:29.985650191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:29.986932113Z caller=checkpoint.go:372 msg="writing series" size="486 kB"
level=debug ts=2026-05-07T15:40:33.319675887Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-07T15:40:34.98627398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:40:35.166875869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:46532"
level=debug ts=2026-05-07T15:40:35.378868409Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:40:35.905238535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:40:35.914685936Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:40:36.652760154Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:40:39.985893718Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:40:39.985918066Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:40:39.985926676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:40:39.985951637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:39.985986678Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=debug ts=2026-05-07T15:40:43.31953074Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-07T15:40:44.983801516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:40:44.98588739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-07T15:40:44.990908242Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:40:46.653547367Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:40:46.653569086Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=8
level=debug ts=2026-05-07T15:40:48.880207806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:40:49.985610237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:49.986865291Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-07T15:40:50.190036081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:40:50.194558389Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-07T15:40:50.30465194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:40:53.320105663Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:40:54.986403064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:56.653424452Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-07T15:40:59.985829141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:40:59.986123186Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-07T15:41:03.319185676Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-07T15:41:03.68630396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:41:03.693811419Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:41:04.723767919Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:41:04.98606804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:06.653635468Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-07T15:41:06.792083714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:41:06.798090622Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:41:06.798113011Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:41:06.798135449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:41:06.803331102Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:41:09.966268214Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:41:09.966859868Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:41:09.986343511Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:41:09.986364388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:41:09.986371306Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:41:09.986379431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:10.042929226Z caller=checkpoint.go:372 msg="writing series" size="51 MB"
level=debug ts=2026-05-07T15:41:10.042960852Z caller=checkpoint.go:384 msg="flushing series" totalSize="52 MB" series=7
level=info ts=2026-05-07T15:41:11.517640435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:41:11.524731139Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:41:11.524755196Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:41:11.524776923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:41:11.531063373Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:41:13.320992801Z caller=checkpoint.go:372 msg="writing series" size="699 kB"
level=info ts=2026-05-07T15:41:13.879893378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:41:13.88805822Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:41:14.986560823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:16.653963499Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-07T15:41:16.653986557Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-07T15:41:17.165923864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:41:17.172104927Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:41:19.985560251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:19.98666966Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-07T15:41:20.30588763Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:41:23.31931124Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-07T15:41:24.985890788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:41:25.854220262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:41:25.867285248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:41:26.443766214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:41:26.451157941Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:41:26.65279642Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=info ts=2026-05-07T15:41:26.891175026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:41:26.898721901Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:41:26.898745752Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:41:26.898773257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:41:26.907177756Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:41:29.986189377Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-07T15:41:29.98624015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:31.922715724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:41:33.319528907Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=debug ts=2026-05-07T15:41:34.784393666Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:41:34.985937728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:36.65252357Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-07T15:41:39.986422955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:41:39.986450946Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:41:39.986458371Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:41:39.98645428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:39.986479831Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-07T15:41:42.950380009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:41:42.959151657Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:41:43.319316163Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-07T15:41:44.985929406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:41:45.082296388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:41:45.088638171Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:41:46.52391139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:41:46.528374378Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:41:46.652853725Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-07T15:41:49.985566987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:49.986678285Z caller=checkpoint.go:372 msg="writing series" size="642 B"
ts=2026-05-07T15:41:50.308596943Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:41:53.319371122Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-07T15:41:54.98628889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:56.654609982Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-07T15:41:56.654631171Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=12
level=info ts=2026-05-07T15:41:59.277161604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:41:59.290093922Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:41:59.986135866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:41:59.986147948Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:42:00.273402965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:42:00.278874396Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:42:02.056518524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:42:02.063499963Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:42:03.320358821Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=warn ts=2026-05-07T15:42:04.985679255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:42:05.174339192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:48260"
level=debug ts=2026-05-07T15:42:06.652703176Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-07T15:42:08.442589798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:42:08.980464096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="400 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:42:08.987078795Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:42:09.50858286Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:42:09.508652109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:42:09.513534144Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:42:09.966352827Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:42:09.966413505Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:42:09.985882418Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:42:09.985896104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:09.985910636Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:42:09.985924056Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:42:09.985985636Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=debug ts=2026-05-07T15:42:13.319698661Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-07T15:42:14.639739183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:42:14.64762179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:42:14.985534702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:16.653557528Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-07T15:42:19.986051673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:19.986145184Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
ts=2026-05-07T15:42:20.310471992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:42:20.460151168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:42:20.465652835Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:42:23.319668919Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-07T15:42:23.319691643Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-07T15:42:24.985726915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:26.653163008Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-07T15:42:29.986012698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:29.986035054Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-07T15:42:33.319334716Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-07T15:42:34.985591936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:42:35.658605326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:42:35.664490746Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:42:36.653368054Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-07T15:42:39.986363846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:42:39.986371839Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:42:39.986411855Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:42:39.986423148Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:42:39.98659639Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=info ts=2026-05-07T15:42:40.963463384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:42:40.970042237Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:42:43.248675558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:42:43.254997791Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:42:43.319668512Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T15:42:43.319691037Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-05-07T15:42:43.521894693Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:42:44.426397112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:42:44.433178509Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:42:44.986505847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:46.653005105Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-07T15:42:49.986051807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:49.986142866Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
ts=2026-05-07T15:42:50.311629955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:42:53.165315628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:42:53.172868949Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:42:53.319114475Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=warn ts=2026-05-07T15:42:54.986201827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:42:56.417531164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:42:56.42147172Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:42:56.653192335Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-07T15:42:59.98561899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:42:59.986883303Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-07T15:43:01.246298602Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:43:03.320132218Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T15:43:04.986240717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:06.65267019Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-07T15:43:08.518320387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:43:08.527094997Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:43:09.966245921Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:43:09.96673896Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T15:43:09.985772034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:43:09.985811843Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:43:09.985842461Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:43:09.985850442Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:43:09.98597964Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-07T15:43:09.985990348Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=debug ts=2026-05-07T15:43:13.319809169Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-07T15:43:14.256529659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:43:14.265240987Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:43:14.9858103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:15.545590681Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:43:16.652796228Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-07T15:43:17.120557183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:43:17.124687397Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:43:19.985935128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:19.985982679Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-07T15:43:20.314349292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:43:23.319534039Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=info ts=2026-05-07T15:43:23.328219783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:43:23.335793323Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:43:24.986365737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:26.653332Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-07T15:43:26.653355992Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
ts=2026-05-07T15:43:26.889797426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60036"
ts=2026-05-07T15:43:29.19736751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46138"
level=warn ts=2026-05-07T15:43:29.985624591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:29.98707298Z caller=checkpoint.go:372 msg="writing series" size="936 kB"
level=info ts=2026-05-07T15:43:33.056748673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:43:33.062773277Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:43:33.319342505Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-07T15:43:34.985628231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:36.653389739Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-07T15:43:36.653411485Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-07T15:43:39.986049693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:43:39.986078377Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:43:39.986086483Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:43:39.986112446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:39.986352861Z caller=checkpoint.go:372 msg="writing series" size="495 kB"
level=info ts=2026-05-07T15:43:41.392184636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:43:41.400040059Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:43:42.821023835Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:43:43.31938431Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-07T15:43:44.986334411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:46.65350298Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-07T15:43:46.676666081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:43:46.684776713Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:43:49.98562241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:49.986826022Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-07T15:43:49.986838327Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-07T15:43:50.316972857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:43:50.677566685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:43:50.681897165Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:43:50.681918603Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:43:50.68194315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:43:50.685125485Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:43:53.321450732Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-07T15:43:53.321475083Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=warn ts=2026-05-07T15:43:54.986048244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:56.653642567Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-07T15:43:56.653667161Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=1
ts=2026-05-07T15:43:56.890830825Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53250"
level=debug ts=2026-05-07T15:43:57.449441991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:43:57.451614136Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:43:57.526859446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:43:57.534096049Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:43:59.985879233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:43:59.985945156Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-07T15:44:00.081236418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:44:00.089819719Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:44:03.319276152Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=info ts=2026-05-07T15:44:03.407678761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:44:03.412699185Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-07T15:44:03.412721241Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:44:03.412744767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:44:03.416331436Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:44:04.082448719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:44:04.086231236Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:44:04.986313951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:44:05.726117169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:44:05.732509517Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:44:06.653510112Z caller=checkpoint.go:372 msg="writing series" size="547 kB"
level=info ts=2026-05-07T15:44:09.966349768Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:44:09.966367813Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:44:09.985708118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:44:09.98573366Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:44:09.985743378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:44:09.985746861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:09.985915092Z caller=checkpoint.go:372 msg="writing series" size="444 kB"
level=debug ts=2026-05-07T15:44:09.985927344Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-07T15:44:12.887085897Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:44:13.319427936Z caller=checkpoint.go:372 msg="writing series" size="356 kB"
level=warn ts=2026-05-07T15:44:14.986174046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:16.652862481Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-07T15:44:19.986459355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:19.990874429Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-07T15:44:19.990890793Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=3
ts=2026-05-07T15:44:20.319043569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:44:22.663138595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:44:22.6692437Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:44:23.320174121Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:44:24.98596096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:26.653601681Z caller=checkpoint.go:372 msg="writing series" size="690 kB"
level=info ts=2026-05-07T15:44:27.473226114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:44:27.483551297Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=error ts=2026-05-07T15:44:27.483582121Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:44:27.48360632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:44:27.49057508Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:44:28.481003412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:44:28.48495639Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T15:44:29.200954928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50098"
level=warn ts=2026-05-07T15:44:29.985499576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:29.986897671Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T15:44:29.986916783Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-07T15:44:30.855318583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:44:30.860747709Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:44:33.319879951Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T15:44:34.986302288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:44:36.653326646Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-07T15:44:37.492699647Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:44:38.613462816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:44:38.621518893Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:44:39.493380668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:44:39.985734176Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:44:39.98573425Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=debug ts=2026-05-07T15:44:39.98576132Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:44:39.985769938Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:44:39.985763226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:44:39.995350829Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-07T15:44:39.996544106Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010835636s
level=info ts=2026-05-07T15:44:40.286078636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:44:40.292198285Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:44:44.985770552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:44:49.985815763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:44:50.320299935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:44:51.964788734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:44:51.97675464Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:44:51.976776365Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:44:51.976811507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:44:51.984048729Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:44:53.832025604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:44:53.838785208Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:44:54.986034911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:44:56.893518397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:52160"
level=info ts=2026-05-07T15:44:58.887985384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:44:58.892848904Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:44:59.986301361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:45:01.215617771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:45:01.221599737Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:45:01.92153982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:45:01.929186889Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:45:02.96662173Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:45:04.98550969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:45:09.966280696Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:45:09.967433318Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=debug ts=2026-05-07T15:45:09.968693338Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T15:45:09.968736204Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975742 n=1
level=debug ts=2026-05-07T15:45:09.96877627Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975742 n=1
level=info ts=2026-05-07T15:45:09.985361818Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:45:09.985488214Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-07T15:45:09.985497095Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:45:09.985499791Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T15:45:09.985514633Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.903µs success=true
level=debug ts=2026-05-07T15:45:09.985519003Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:45:09.985528436Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:45:09.985532597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:09.985753369Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-07T15:45:11.02146903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:45:11.030192527Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:45:13.325828813Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T15:45:13.325852213Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=warn ts=2026-05-07T15:45:14.98614882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:16.653273121Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-07T15:45:19.98601156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:19.998421812Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-07T15:45:19.998445983Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
ts=2026-05-07T15:45:20.321334781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:45:23.320154601Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-07T15:45:24.023069265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:45:24.032634479Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:45:24.764680119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:45:24.771168857Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:45:24.985694399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:45:25.941677219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:45:25.945409238Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:45:26.653405454Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-07T15:45:29.985518602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:29.986876717Z caller=checkpoint.go:372 msg="writing series" size="541 kB"
level=debug ts=2026-05-07T15:45:32.739952216Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:45:33.319821714Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-07T15:45:34.985911641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:35.475574163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:45:36.652986118Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:45:36.897665589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:45:36.905695629Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:45:39.986246552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:45:39.986277685Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:45:39.986301608Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:45:39.986311009Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:45:39.98634555Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-07T15:45:43.319293974Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-07T15:45:44.985528622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:45:45.829513904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:45:45.83529573Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:45:46.653805478Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T15:45:46.653827821Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=8
level=warn ts=2026-05-07T15:45:49.985604633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:49.986933237Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
ts=2026-05-07T15:45:50.323315772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:45:51.276898993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:45:51.284356588Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:45:52.214955732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:45:52.222237449Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:45:53.319391781Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:45:54.985718777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:56.653250346Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-07T15:45:57.565011221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:45:57.569709461Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:45:59.985981013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:45:59.986250067Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-07T15:46:03.31931567Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-07T15:46:04.52960274Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:46:04.985964652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:46:05.198529165Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37992"
level=debug ts=2026-05-07T15:46:06.653534488Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-07T15:46:09.967021855Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:46:09.967684541Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:46:09.986074085Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:46:09.986081236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:09.986102523Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:46:09.986116865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:46:10.038577734Z caller=checkpoint.go:372 msg="writing series" size="52 MB"
level=debug ts=2026-05-07T15:46:10.038610163Z caller=checkpoint.go:384 msg="flushing series" totalSize="53 MB" series=7
level=debug ts=2026-05-07T15:46:13.320857312Z caller=checkpoint.go:372 msg="writing series" size="802 kB"
level=info ts=2026-05-07T15:46:13.826448997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:46:13.832973767Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:46:14.986410194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:16.653796909Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-07T15:46:16.653820278Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-07T15:46:16.711671535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:46:16.718612265Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:46:19.986318424Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-07T15:46:19.986313433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:46:20.325291238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:46:22.900379328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:46:22.904672339Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:46:23.319993332Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-07T15:46:24.012991428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:46:24.02028198Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:46:24.985699529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:26.653476054Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-07T15:46:29.986020413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:29.986078437Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-07T15:46:31.344574315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:46:31.353723761Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:46:33.319408253Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-07T15:46:34.985730681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:46:35.201336599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57518"
level=debug ts=2026-05-07T15:46:36.652801276Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-05-07T15:46:37.28176935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:46:38.723551423Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:46:39.985924304Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:46:39.985931424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:39.985948417Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:46:39.9859569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:46:39.985999105Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-07T15:46:43.319308748Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-07T15:46:44.985877993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:46.653305851Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-07T15:46:47.170279827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:46:47.177648125Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:46:49.394847329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:46:49.402872576Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:46:49.98549239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:49.986587155Z caller=checkpoint.go:372 msg="writing series" size="642 B"
ts=2026-05-07T15:46:50.326720703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:46:53.319717099Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-07T15:46:54.986223454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:46:56.12314365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:46:56.127978349Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:46:56.467113221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:46:56.472541235Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:46:56.654609826Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-07T15:46:56.654639761Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=12
ts=2026-05-07T15:46:56.90288876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43328"
level=warn ts=2026-05-07T15:46:59.98579706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:46:59.985846471Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-07T15:47:03.320156823Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=debug ts=2026-05-07T15:47:04.814159842Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:47:04.950321606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:47:04.955042995Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:47:04.986363595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:06.653293445Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:47:09.966184308Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:47:09.967802612Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:47:09.986031268Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:47:09.986051116Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:47:09.986060179Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:47:09.986063822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:09.98617507Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=debug ts=2026-05-07T15:47:13.31968519Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:47:14.985993347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:47:15.062270177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:47:15.069817671Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:47:16.396432122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:47:16.402698811Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:47:16.402722937Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:47:16.402747867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:47:16.408377385Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:47:16.653320638Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-07T15:47:19.320016329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:47:19.329499551Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:47:19.986429847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:19.986505927Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
ts=2026-05-07T15:47:20.328942715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:47:23.319355069Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-07T15:47:23.319376865Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-07T15:47:24.985893178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:26.653321204Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-07T15:47:27.701844896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-07T15:47:29.211738853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33752"
level=warn ts=2026-05-07T15:47:29.98599327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:29.986028691Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-07T15:47:32.5792212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:47:32.587928313Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:47:33.319909926Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-07T15:47:34.158627056Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:47:34.985614105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:36.653142648Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-07T15:47:39.986026599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:47:39.986051367Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:47:39.986059247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:47:39.986086722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:39.986116658Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-05-07T15:47:41.027645477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:47:41.034986785Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:47:43.320163874Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T15:47:43.320188061Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=warn ts=2026-05-07T15:47:44.986515631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:47:45.007676157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:47:45.013486879Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:47:45.810271419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:47:45.818744953Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:47:46.653161685Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-07T15:47:49.462208535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:47:49.467097504Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:47:49.986368916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:49.98646692Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-07T15:47:50.332287289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:47:53.319915176Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-07T15:47:54.631710174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:47:54.63629693Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:47:54.985814587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:56.653180495Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
ts=2026-05-07T15:47:56.906075071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33456"
level=warn ts=2026-05-07T15:47:59.985503603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:47:59.986626056Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-07T15:48:02.501171606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:48:02.507727277Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:48:02.507755205Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:48:02.507780518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:48:02.514145673Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:48:03.319654961Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T15:48:04.986028381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:48:05.210476868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:52740"
level=info ts=2026-05-07T15:48:05.349475959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:48:05.356451806Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:48:06.652414931Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:48:07.70660651Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:48:08.652754395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:48:08.654318774Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:48:09.966190417Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:48:09.966238666Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:48:09.985498113Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:48:09.985522809Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:48:09.985530225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:48:09.985528182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:09.986876572Z caller=checkpoint.go:372 msg="writing series" size="344 kB"
level=debug ts=2026-05-07T15:48:13.319440219Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=debug ts=2026-05-07T15:48:13.319465794Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-07T15:48:14.985886496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:16.653329425Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-07T15:48:17.484892323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:48:17.496021633Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:48:18.754516867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:48:18.762783357Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:48:19.986314339Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-07T15:48:19.986340901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:48:20.334498939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:48:23.320202288Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=warn ts=2026-05-07T15:48:24.985822934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:48:25.590198219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:48:25.59665953Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:48:25.596684563Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:48:25.596717829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:48:25.602435464Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:48:26.653748523Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-07T15:48:26.653771372Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-07T15:48:29.985534119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:29.987671363Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=debug ts=2026-05-07T15:48:33.319863304Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-05-07T15:48:33.820320439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:48:33.828358708Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:48:33.828383026Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:48:33.828407603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:48:33.835522208Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:48:34.986115749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:36.652924281Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-07T15:48:36.652950295Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-07T15:48:38.310822065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:48:38.320141877Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:48:38.801522141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:48:38.807586646Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:48:39.019286316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:48:39.023382081Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:48:39.985874069Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:48:39.985880563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:39.985898577Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:48:39.985907122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:48:39.986174086Z caller=checkpoint.go:372 msg="writing series" size="368 kB"
level=debug ts=2026-05-07T15:48:40.049353954Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:48:43.320019751Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-07T15:48:44.986028523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:46.653585266Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-07T15:48:48.550390435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:48:48.554778992Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:48:48.701323414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:48:48.708503941Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:48:49.325129781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:48:49.986351347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:49.986457287Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-05-07T15:48:50.335678152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T15:48:53.133118125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:48:53.140386139Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:48:53.322357624Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-07T15:48:53.322388694Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=5
level=warn ts=2026-05-07T15:48:54.985638544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:56.653982762Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-07T15:48:56.654005436Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=1
level=warn ts=2026-05-07T15:48:59.985565674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:48:59.986793548Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=info ts=2026-05-07T15:49:03.103673833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:49:03.110483383Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:49:03.320043473Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=warn ts=2026-05-07T15:49:04.985479928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:49:05.21673425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37146"
level=debug ts=2026-05-07T15:49:06.653193164Z caller=checkpoint.go:372 msg="writing series" size="547 kB"
level=debug ts=2026-05-07T15:49:09.826165046Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:49:09.966642606Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:49:09.967205244Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:49:09.980799255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:49:09.985788069Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:49:09.985837231Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:49:09.98584614Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:49:09.985857027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:09.986049854Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=debug ts=2026-05-07T15:49:09.986063929Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=error ts=2026-05-07T15:49:09.988676704Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T15:49:12.295308753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="77 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:49:12.301904773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:49:12.802482976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:49:12.809402935Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:49:13.32002619Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=warn ts=2026-05-07T15:49:14.985747476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:16.652723409Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-07T15:49:19.985487068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:19.988595463Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-07T15:49:19.988609974Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.3 MB" series=3
ts=2026-05-07T15:49:20.337007218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:49:22.628809833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:49:22.633162423Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:49:23.319093229Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:49:24.985809321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:26.652834434Z caller=checkpoint.go:372 msg="writing series" size="725 kB"
level=warn ts=2026-05-07T15:49:29.98611102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:29.986467058Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T15:49:29.986482662Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=debug ts=2026-05-07T15:49:33.320168441Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T15:49:33.845473469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:49:33.858854599Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:49:34.668338596Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:49:34.985806578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:49:35.219978573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37962"
level=debug ts=2026-05-07T15:49:36.652471159Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-07T15:49:38.131067727Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:49:39.985778978Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T15:49:39.985792865Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:49:39.985797774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:49:39.985819456Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:49:39.9858367Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:49:39.990167147Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-07T15:49:39.991412482Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005765836s
level=info ts=2026-05-07T15:49:42.987430462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:49:42.993104364Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:49:44.986054058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:49:45.18386306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:49:45.189753539Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:49:48.062641513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:49:48.068984106Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:49:49.98579041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:49:50.339632915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:49:52.548865449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:49:52.555085295Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:49:54.985633016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:49:56.628488822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:49:56.637510986Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-07T15:49:56.914395687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36418"
level=warn ts=2026-05-07T15:49:59.985823055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:50:04.986437322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:50:05.232064184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:50:05.240109645Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:50:06.057226716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:06.060932619Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:06.060952981Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:50:06.060975648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:06.064021358Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:50:07.886458473Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:50:09.966208666Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:50:09.966823121Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:50:09.985771846Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:50:09.98581159Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:50:09.985828618Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:50:09.985822219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:50:09.985836749Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:50:09.986073442Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-07T15:50:09.986083619Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:50:09.986106156Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.122µs success=true
level=debug ts=2026-05-07T15:50:09.986400935Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-07T15:50:13.324783011Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T15:50:13.32480896Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=warn ts=2026-05-07T15:50:14.985534334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:16.653961546Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=warn ts=2026-05-07T15:50:19.986445676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:19.992927588Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-07T15:50:19.992945308Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
ts=2026-05-07T15:50:20.341312555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:50:23.320537463Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-07T15:50:23.737227061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:23.741311613Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:50:24.985835711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:25.786976523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:50:26.653498218Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T15:50:26.915749919Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35964"
level=info ts=2026-05-07T15:50:27.862282264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:50:27.872170287Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
ts=2026-05-07T15:50:29.222820434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51128"
level=warn ts=2026-05-07T15:50:29.986141554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:29.987528901Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
level=info ts=2026-05-07T15:50:32.639806615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:50:32.645909141Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:50:33.319913238Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=info ts=2026-05-07T15:50:34.154312881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:50:34.15893393Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:50:34.986245792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:50:35.225928213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47104"
level=debug ts=2026-05-07T15:50:36.653549666Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:50:39.986240122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:50:39.986284012Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:50:39.986294452Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:50:39.98630164Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:50:39.986481418Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-05-07T15:50:40.591935823Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:50:43.320293664Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-07T15:50:43.782721933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:50:43.790498978Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:50:44.986075773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:50:46.588303195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:50:46.592359294Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:50:46.654212542Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T15:50:46.654235801Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=8
level=info ts=2026-05-07T15:50:47.60013629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:50:47.606286962Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:50:47.697458839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:50:47.70304812Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:50:49.985894089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:49.987077076Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-05-07T15:50:50.343132248Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:50:51.907202188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:50:51.915035484Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:50:53.319927708Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:50:54.985664582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:56.653823952Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
ts=2026-05-07T15:50:56.916539026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47682"
level=warn ts=2026-05-07T15:50:59.986388664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:50:59.986582667Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-07T15:51:03.320356881Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-07T15:51:03.87794987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:51:04.985648571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:06.653373697Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-07T15:51:07.612091661Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:51:09.966226278Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:51:09.967296988Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:51:09.985588239Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:51:09.98559365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:09.985618294Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:51:09.985626237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:51:10.051469786Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-07T15:51:10.051499246Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=7
level=debug ts=2026-05-07T15:51:13.320826422Z caller=checkpoint.go:372 msg="writing series" size="933 kB"
level=warn ts=2026-05-07T15:51:14.986150766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:51:15.382176168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:51:15.39104063Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=error ts=2026-05-07T15:51:15.391067119Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:51:15.391092821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:51:15.397904965Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:51:16.654202917Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-07T15:51:16.654230599Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=warn ts=2026-05-07T15:51:19.985549508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:19.986679189Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-07T15:51:20.344929887Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:51:23.320349723Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=info ts=2026-05-07T15:51:23.670460363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:51:23.676857566Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:51:24.986192493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:51:26.432460023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:51:26.440277506Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:51:26.441246713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:51:26.445719128Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-07T15:51:26.445739961Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:51:26.445760766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:51:26.449871783Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:51:26.65359341Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
ts=2026-05-07T15:51:29.227655445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39158"
level=warn ts=2026-05-07T15:51:29.985570039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:29.98682255Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-07T15:51:32.248817995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:51:32.257095707Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:51:32.25711975Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:51:32.257155368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:51:32.264671918Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:51:33.082271261Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:51:33.320004359Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-07T15:51:34.361479127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:51:34.365689675Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:51:34.98615625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:36.653107843Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T15:51:39.722003761Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:51:39.723807288Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:51:39.985782836Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:51:39.98578675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:39.985807749Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:51:39.985815517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:51:39.986962382Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-07T15:51:40.244950606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:51:40.249880748Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:51:40.7229805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:51:40.729566923Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:51:43.320039971Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-07T15:51:44.392127422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:51:44.397362161Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:51:44.986374125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:46.654087807Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-07T15:51:49.227229874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="424 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:51:49.236226742Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:51:49.986283426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:49.986379187Z caller=checkpoint.go:372 msg="writing series" size="657 B"
ts=2026-05-07T15:51:50.346599946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T15:51:50.963957745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:51:50.970789479Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:51:53.320326693Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-07T15:51:54.986036753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:56.654348127Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-07T15:51:56.654381401Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=12
ts=2026-05-07T15:51:56.919355142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55698"
level=info ts=2026-05-07T15:51:57.682973772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:51:57.689908712Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:51:59.986414269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:51:59.986484052Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-07T15:52:01.376915612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:52:01.382443501Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:52:03.32113572Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=debug ts=2026-05-07T15:52:04.349063681Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:52:04.986055717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:52:05.233799301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34384"
level=debug ts=2026-05-07T15:52:06.653668892Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:52:09.966806086Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:52:09.967436467Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:52:09.986209788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:52:09.986235155Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:52:09.986243169Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:52:09.986235094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:09.986522724Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
level=info ts=2026-05-07T15:52:12.638141944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:52:12.645575612Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:52:13.320402404Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=warn ts=2026-05-07T15:52:14.985657724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:16.65391788Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=debug ts=2026-05-07T15:52:18.650306991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T15:52:19.985883824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:19.987246723Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=debug ts=2026-05-07T15:52:19.987282965Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
ts=2026-05-07T15:52:20.348809251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:52:20.435018035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="448 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:52:20.444678598Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:52:23.320016263Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=info ts=2026-05-07T15:52:24.33592264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:52:24.340121653Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:52:24.986436095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:26.653977734Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T15:52:29.986102218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:29.987211102Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-07T15:52:31.391441486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:52:31.397623593Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:52:31.650391621Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:52:33.320131369Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-07T15:52:33.699310661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:52:33.706514412Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:52:34.986293234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:52:35.236903477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38724"
level=debug ts=2026-05-07T15:52:36.65311551Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-07T15:52:37.875570652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:52:37.884053863Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:52:39.986021097Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:52:39.986019816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:39.98604907Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:52:39.986059691Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:52:39.987227273Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=debug ts=2026-05-07T15:52:43.320939689Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T15:52:43.320963897Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=warn ts=2026-05-07T15:52:44.985966021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:46.6531185Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-07T15:52:49.21238086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:52:49.221636832Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:52:49.986377512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:49.986460394Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
ts=2026-05-07T15:52:50.350596272Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:52:51.259476157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:52:51.267275762Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:52:53.320361433Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=warn ts=2026-05-07T15:52:54.986298335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:56.653192892Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
ts=2026-05-07T15:52:59.234170775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48598"
level=warn ts=2026-05-07T15:52:59.985493326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:52:59.986595959Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-07T15:53:01.522921416Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:53:03.320301051Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T15:53:04.985483539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:53:05.190886549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:53:05.197423816Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:53:06.6531959Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-07T15:53:08.012068657Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:53:09.966192767Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:53:09.96677805Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T15:53:09.985584791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:53:09.985593008Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:53:09.985615961Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:53:09.985623801Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:53:09.986930296Z caller=checkpoint.go:372 msg="writing series" size="480 kB"
level=debug ts=2026-05-07T15:53:09.986942406Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-07T15:53:11.452669123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:53:11.458998457Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:53:13.268152071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="462 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:53:13.27785259Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:53:13.319923914Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-07T15:53:14.985871785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:16.653519476Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T15:53:19.986312228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:19.986332717Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T15:53:20.215716012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:53:20.224213263Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-07T15:53:20.352496139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:53:23.320752967Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=info ts=2026-05-07T15:53:23.471998494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:53:23.481540338Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:53:24.986464075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:53:25.308306356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:53:25.314562339Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:53:26.65334281Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-07T15:53:26.653365277Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
ts=2026-05-07T15:53:26.924106727Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46750"
level=debug ts=2026-05-07T15:53:28.928046239Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:53:29.2353738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50288"
level=warn ts=2026-05-07T15:53:29.985966214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:29.987393496Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-07T15:53:33.320009006Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-05-07T15:53:33.320032021Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-07T15:53:34.985774791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:53:35.241967475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:54270"
level=debug ts=2026-05-07T15:53:36.653375923Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-05-07T15:53:39.985744223Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:53:39.985754515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:39.985767506Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:53:39.985775433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:53:39.987109962Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=debug ts=2026-05-07T15:53:43.319944342Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-07T15:53:43.319979619Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=warn ts=2026-05-07T15:53:44.98592998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:46.653763689Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-07T15:53:47.470083176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:53:47.478060837Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:53:49.145215032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:53:49.153715388Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:53:49.98637234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:49.986507294Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-05-07T15:53:50.353921641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:53:53.323124959Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T15:53:53.323148311Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=3
level=debug ts=2026-05-07T15:53:54.34306152Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:53:54.986067267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:53:55.677665033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:53:55.682663383Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:53:56.654168496Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-07T15:53:56.654191906Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=1
level=info ts=2026-05-07T15:53:57.314498273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:53:57.320689219Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:53:57.320715818Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:53:57.320747286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:53:57.32679986Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:53:59.74833365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="462 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:53:59.755724114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:53:59.986463084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:53:59.986642104Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=debug ts=2026-05-07T15:54:03.319901085Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=warn ts=2026-05-07T15:54:04.986421511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:54:05.244715263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47170"
level=debug ts=2026-05-07T15:54:06.653937266Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-07T15:54:09.19764508Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:54:09.304908671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:54:09.312456959Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T15:54:09.966436812Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:54:09.966455834Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:54:09.986480981Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:54:09.986524139Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T15:54:09.986514163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:54:09.986535771Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:54:09.986715717Z caller=checkpoint.go:372 msg="writing series" size="524 kB"
level=debug ts=2026-05-07T15:54:09.986728157Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-05-07T15:54:13.320855386Z caller=checkpoint.go:372 msg="writing series" size="386 kB"
level=info ts=2026-05-07T15:54:13.561558649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:54:13.567655777Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:54:14.986286395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:16.653130026Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-07T15:54:18.673171995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:54:18.678409186Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:54:19.985908623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:19.991206195Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-05-07T15:54:19.99122346Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=3
ts=2026-05-07T15:54:20.35594543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:54:22.670904669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:54:22.678870117Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:54:23.320684391Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:54:24.985673804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:26.654623184Z caller=checkpoint.go:372 msg="writing series" size="871 kB"
level=debug ts=2026-05-07T15:54:29.968190698Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T15:54:29.986093962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:29.987536451Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T15:54:29.987558845Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-05-07T15:54:33.320591936Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T15:54:34.986609358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:36.653246009Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T15:54:36.720488499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:54:36.729654557Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:54:39.986397231Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:54:39.986403921Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=debug ts=2026-05-07T15:54:39.986427046Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:54:39.986435747Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:54:39.986433236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:54:39.990664828Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-07T15:54:39.992175716Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005905706s
level=info ts=2026-05-07T15:54:40.756003664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:54:40.761818833Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:54:44.986538956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:54:46.944401953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:54:46.953743422Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:54:46.9639252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:54:46.972730836Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:54:49.986086496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:54:50.358879898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:54:51.351522397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:54:51.363690356Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T15:54:54.985853856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:54:55.609104702Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T15:54:56.931589712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34376"
ts=2026-05-07T15:54:59.239278576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:44550"
level=warn ts=2026-05-07T15:54:59.985796658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:55:04.357465922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:55:04.36897361Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:55:04.368999391Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:55:04.369029838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:55:04.374966214Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:55:04.985505048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:55:07.861219766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:55:07.868293493Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:55:07.868317522Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:55:07.86835303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:55:07.874588998Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:55:09.131449785Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:55:09.966370951Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:55:09.96639837Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:55:09.985863226Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T15:55:09.985890276Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:55:09.985890716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:09.985910704Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:55:09.985918515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:55:09.986012697Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-07T15:55:09.986021344Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T15:55:09.986038037Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.913µs success=true
level=debug ts=2026-05-07T15:55:09.986315421Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-07T15:55:10.748988172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:55:10.754554471Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:55:10.754583959Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:55:10.754610403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:55:10.759319062Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:55:13.207565367Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T15:55:13.207587597Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=warn ts=2026-05-07T15:55:14.985869983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:55:15.085200723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:55:15.092517597Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:55:16.415301069Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T15:55:19.639838159Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-07T15:55:19.639864867Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=warn ts=2026-05-07T15:55:19.986236877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:55:20.361541175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T15:55:21.934856496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:55:21.942681967Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:55:22.844506802Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-05-07T15:55:23.058929607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:55:23.06415295Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:55:24.520884994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:55:24.527272938Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:55:24.986293314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:26.058557819Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T15:55:29.272873088Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-07T15:55:29.985505006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:30.074048261Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:55:31.693979901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:55:31.698689433Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:55:32.487192557Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-07T15:55:34.986508109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:55:35.253666338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55040"
level=debug ts=2026-05-07T15:55:35.700742992Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-07T15:55:36.734410275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:55:36.742184809Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:55:36.91429527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:55:36.920448217Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:55:38.915418235Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-07T15:55:39.296818405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:55:39.30473387Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T15:55:39.98551635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:55:39.985537431Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:55:39.985560163Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:55:39.985567925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:55:40.289371206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:55:40.295537442Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T15:55:42.129384473Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-07T15:55:44.986228435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:45.344530147Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T15:55:45.344558058Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=8
level=debug ts=2026-05-07T15:55:48.558214387Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=warn ts=2026-05-07T15:55:49.985492706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:55:50.363067509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T15:55:51.6940812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:55:51.703467449Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:55:51.772760204Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-07T15:55:54.986152781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:55:54.986239709Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-07T15:55:58.025712463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:55:58.031822972Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:55:58.201639622Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=warn ts=2026-05-07T15:55:59.985653104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:01.415421715Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-07T15:56:04.629609598Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-07T15:56:04.985650371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:05.569321728Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:56:05.569301037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="48 kB" avg_uncomp="48 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:56:05.572678099Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T15:56:05.86993634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:56:05.873349653Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T15:56:06.170200925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="598 B" avg_uncomp="598 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:56:06.17340124Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T15:56:06.470237151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:56:06.473477805Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T15:56:07.906705389Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-07T15:56:07.906750982Z caller=checkpoint.go:384 msg="flushing series" totalSize="61 MB" series=7
level=debug ts=2026-05-07T15:56:07.962658687Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T15:56:07.964540898Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T15:56:09.966364807Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:56:09.966402599Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:56:09.986021732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:56:09.986047091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:56:09.98605431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:56:09.986111497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:11.059497365Z caller=checkpoint.go:372 msg="writing series" size="867 kB"
level=info ts=2026-05-07T15:56:11.099669087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:56:11.104721066Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:56:14.272928741Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-07T15:56:14.272952122Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-07T15:56:14.986075508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:56:15.861213315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:56:15.865001994Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:56:15.865026492Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:56:15.865053755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:56:15.868147634Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:56:17.009213727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:56:17.012308504Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T15:56:17.487283048Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-07T15:56:19.000459558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:56:19.003383071Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T15:56:19.986285148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:56:20.365466216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T15:56:20.701415911Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-05-07T15:56:20.831704507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:56:20.839549258Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:56:22.157405868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:56:22.160417015Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T15:56:23.40003959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:56:23.407051961Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:56:23.878308788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:56:23.881793445Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T15:56:23.914858776Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-07T15:56:24.986316504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:56:25.709004819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:56:25.72158475Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
ts=2026-05-07T15:56:26.938357861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50646"
level=debug ts=2026-05-07T15:56:27.129422296Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
ts=2026-05-07T15:56:29.244427111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:47888"
level=info ts=2026-05-07T15:56:29.362135089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:56:29.366480236Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:56:29.98639816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:30.344243524Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-07T15:56:31.532683991Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:56:31.532732991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:56:31.536010516Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T15:56:32.594126232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:56:32.599989622Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:56:33.558766266Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-07T15:56:34.986014845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:56:35.259216255Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59354"
level=info ts=2026-05-07T15:56:36.057244722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:56:36.066135041Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:56:36.77222457Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-07T15:56:39.986448289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:56:39.986441983Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:56:39.986471673Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:56:39.986480339Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:56:39.986545357Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-07T15:56:41.989462728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:56:41.996059304Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:56:43.201439826Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-07T15:56:43.249541129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:56:43.253362918Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T15:56:44.710679185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:56:44.714077429Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T15:56:44.986421171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:46.415431467Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-07T15:56:49.124711227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:56:49.128727347Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T15:56:49.477824999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:56:49.481209118Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T15:56:49.629696736Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T15:56:49.985948665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:56:50.367122728Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:56:52.846873489Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-07T15:56:52.847002242Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=12
level=info ts=2026-05-07T15:56:54.303997488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:56:54.308322779Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T15:56:54.986294904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:56:56.057806327Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
ts=2026-05-07T15:56:56.939415872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38318"
ts=2026-05-07T15:56:59.245368179Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39526"
level=debug ts=2026-05-07T15:56:59.27297217Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-07T15:56:59.985798596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:00.76495678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:57:00.769892481Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:57:01.290590111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:57:01.29375035Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T15:57:01.995054863Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:57:02.486464259Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-07T15:57:02.83361033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T15:57:02.833641521Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-07T15:57:02.836865011Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T15:57:03.13365678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T15:57:03.136901242Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T15:57:03.433825796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T15:57:03.437606891Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T15:57:04.985979663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:05.701364604Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=info ts=2026-05-07T15:57:07.114015045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:57:07.119375329Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T15:57:08.915204299Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=info ts=2026-05-07T15:57:09.96618268Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:57:09.966598659Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:57:09.98559527Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:57:09.98560484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:09.98561612Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:57:09.985626328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:57:10.789760398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:57:10.797135203Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:57:12.129800666Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-07T15:57:13.403412301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T15:57:13.407329137Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T15:57:13.490551163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:57:13.497511157Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:57:14.98565759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:15.043888462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T15:57:15.047438346Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T15:57:15.34393475Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=info ts=2026-05-07T15:57:17.715219503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:57:17.721574037Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:57:18.55797468Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=debug ts=2026-05-07T15:57:18.557998155Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-07T15:57:19.985486031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:57:20.368845612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:57:21.250443864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:57:21.259095903Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:57:21.772663932Z caller=checkpoint.go:372 msg="writing series" size="336 kB"
level=info ts=2026-05-07T15:57:21.92979754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T15:57:21.933376064Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T15:57:24.985519999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:57:24.986653923Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-07T15:57:26.940483611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:33568"
level=debug ts=2026-05-07T15:57:28.201284776Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-07T15:57:29.985992995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:30.738982641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:57:30.742773681Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T15:57:31.41569524Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T15:57:31.794780664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:57:31.802456053Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:57:34.629701528Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=warn ts=2026-05-07T15:57:34.985603785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:35.22138462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T15:57:35.224709224Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-07T15:57:35.265664628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:48306"
level=debug ts=2026-05-07T15:57:35.758617642Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:57:35.758668999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="126 kB" avg_uncomp="126 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T15:57:35.762394637Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T15:57:37.634873018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:57:37.639943775Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T15:57:37.843594815Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
level=info ts=2026-05-07T15:57:39.54430278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:57:39.549201447Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:57:39.985984946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:39.985992629Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:57:39.98601565Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:57:39.98602544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T15:57:40.566321192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T15:57:40.569487078Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T15:57:41.059003103Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T15:57:41.059028192Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=7
level=debug ts=2026-05-07T15:57:44.272024112Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-07T15:57:44.986559475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:44.994698313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:57:44.998522657Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:57:45.512071311Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T15:57:47.486405693Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-07T15:57:49.276908372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:57:49.280150864Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T15:57:49.98611629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:50.237921521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T15:57:50.241229082Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T15:57:50.371188909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:57:50.700555898Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=debug ts=2026-05-07T15:57:53.915779566Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-07T15:57:54.985854793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:57:55.922203328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:57:55.929658241Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:57:56.561545498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:57:56.56448845Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T15:57:56.784309567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:57:56.78889504Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-07T15:57:56.788919727Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:57:56.788951563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:57:56.792625115Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-07T15:57:56.941892801Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47962"
level=debug ts=2026-05-07T15:57:57.129268203Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T15:57:58.10794124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T15:57:58.110904304Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-07T15:57:59.250154562Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50150"
level=warn ts=2026-05-07T15:57:59.985827767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:00.343956379Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T15:58:01.326307043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:58:01.330177478Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T15:58:01.418538026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:58:01.426347628Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=error ts=2026-05-07T15:58:01.426366432Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:58:01.426399468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:58:01.43353195Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:58:01.456627859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:58:01.464558766Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:58:01.464574773Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T15:58:01.727232672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:58:01.734853861Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:58:03.557985119Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-07T15:58:04.986515949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:06.773245108Z caller=checkpoint.go:372 msg="writing series" size="371 kB"
level=debug ts=2026-05-07T15:58:06.773287731Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-05-07T15:58:09.722388113Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T15:58:09.966285817Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:58:09.966527839Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:58:09.985612121Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:58:09.985620686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:09.98563427Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:58:09.985642249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:58:09.986762751Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=info ts=2026-05-07T15:58:11.850646054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:58:11.861382362Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T15:58:13.200918569Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-07T15:58:14.316852442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:58:14.321981353Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T15:58:14.985989919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:15.08828647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T15:58:15.092851499Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T15:58:15.5868627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:58:15.594276329Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T15:58:16.415629996Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T15:58:16.685483199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:58:16.691509233Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T15:58:18.051317934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:58:18.058187801Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:58:19.629976787Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=warn ts=2026-05-07T15:58:19.985577496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:58:20.372820241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T15:58:21.336906377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:58:21.34200883Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T15:58:22.844204492Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-07T15:58:22.844227455Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-07T15:58:22.992626966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T15:58:22.996025961Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T15:58:24.985655652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:25.379983715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:58:25.383278512Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T15:58:26.058362305Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-07T15:58:29.272821229Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T15:58:29.272844567Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=warn ts=2026-05-07T15:58:29.986096931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:31.175391471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:58:31.179874156Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:58:32.486316245Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-07T15:58:34.986010747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:35.158460313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:58:35.162195671Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T15:58:35.701369156Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-05-07T15:58:36.150327735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:58:36.157147956Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:58:36.921238571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T15:58:36.92474526Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T15:58:37.415389793Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:58:38.916038488Z caller=checkpoint.go:372 msg="writing series" size="388 kB"
level=warn ts=2026-05-07T15:58:39.985597983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:39.985602745Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:58:39.98562713Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:58:39.985635681Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:58:42.129333841Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-07T15:58:42.560872004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:58:42.566707023Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T15:58:44.744197374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:58:44.750086814Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T15:58:44.985818125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:58:44.995981904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:58:44.999863267Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T15:58:45.344447086Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-07T15:58:45.344468728Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-05-07T15:58:45.825021124Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:58:48.442474945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:58:48.445979252Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T15:58:48.558427833Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-07T15:58:48.764019028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T15:58:48.767276217Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T15:58:48.845477115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:58:48.853419068Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T15:58:49.986427572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:58:50.375165289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T15:58:51.774616707Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-07T15:58:51.774639547Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=2
level=warn ts=2026-05-07T15:58:54.98560454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:58:54.987603874Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T15:58:54.987618786Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=info ts=2026-05-07T15:58:56.112916581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:58:56.116425613Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-07T15:58:56.945396623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42980"
level=info ts=2026-05-07T15:58:58.1171992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:58:58.124872541Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T15:58:58.201380561Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=warn ts=2026-05-07T15:58:59.986141785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:59:01.415213049Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=debug ts=2026-05-07T15:59:04.372112333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:59:04.629685778Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-07T15:59:04.986559844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:59:05.274335504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47264"
level=debug ts=2026-05-07T15:59:07.843725466Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=info ts=2026-05-07T15:59:09.966282736Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T15:59:09.968244605Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T15:59:09.986144439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T15:59:09.986143996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:59:09.986166977Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:59:09.986174885Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T15:59:11.058730776Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=debug ts=2026-05-07T15:59:11.058754904Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-07T15:59:13.482463193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T15:59:13.488876815Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T15:59:13.889990709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:59:13.89468181Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T15:59:14.273056208Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
level=warn ts=2026-05-07T15:59:14.985492874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:59:15.025715785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T15:59:15.032554851Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T15:59:17.486451194Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-07T15:59:18.601517398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T15:59:18.605120232Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T15:59:18.672307095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T15:59:18.677679967Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T15:59:19.986058351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:59:20.376268293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T15:59:20.707280829Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-07T15:59:20.707305738Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.6 MB" series=3
level=info ts=2026-05-07T15:59:23.155516494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T15:59:23.159301025Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T15:59:23.915341228Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T15:59:24.985842065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:59:27.129704783Z caller=checkpoint.go:372 msg="writing series" size="957 kB"
level=warn ts=2026-05-07T15:59:29.985815276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:59:30.194291336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T15:59:30.198547475Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T15:59:30.344305762Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T15:59:30.34432837Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-07T15:59:31.630196603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T15:59:31.63514763Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T15:59:31.956940164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T15:59:31.963458321Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T15:59:32.491936464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T15:59:32.499464756Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T15:59:32.848513651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T15:59:32.85223139Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T15:59:33.013916547Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T15:59:33.558310352Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T15:59:34.543775413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T15:59:34.547058016Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T15:59:34.986090411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:59:35.277330578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:44702"
level=debug ts=2026-05-07T15:59:36.375834257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T15:59:36.744806449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T15:59:36.74807721Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T15:59:36.772466901Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T15:59:38.867319799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T15:59:38.870361335Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T15:59:39.985837854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T15:59:39.985861906Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T15:59:39.985869699Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T15:59:39.985863392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T15:59:39.986935724Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T15:59:39.989321915Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-07T15:59:39.991129876Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004959749s
level=info ts=2026-05-07T15:59:40.996393972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T15:59:41.004499996Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T15:59:44.699062517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T15:59:44.702275719Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T15:59:44.986012556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-07T15:59:49.98630542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T15:59:50.3781527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=warn ts=2026-05-07T15:59:54.98589473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T15:59:55.466301941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T15:59:55.472682134Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T15:59:58.009641982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T15:59:58.016902854Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T15:59:59.986118429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:00.433043138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:00:00.436543907Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:00:01.919271474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:00:01.925388438Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:00:03.510901508Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:00:04.98574895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:06.038233204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:00:06.0443161Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:00:09.893016205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:00:09.897411441Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:00:09.966185966Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:00:09.967086101Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=debug ts=2026-05-07T16:00:09.968370382Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T16:00:09.968410841Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975743 n=1
level=debug ts=2026-05-07T16:00:09.96845015Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975743 n=1
level=info ts=2026-05-07T16:00:09.985724259Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T16:00:09.985752316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:09.985773438Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:00:09.985793654Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:00:09.985803944Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:00:09.985822422Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-07T16:00:09.985833654Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:00:09.985853837Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.914µs success=true
level=debug ts=2026-05-07T16:00:09.9861127Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-07T16:00:13.20763888Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T16:00:13.20766434Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=debug ts=2026-05-07T16:00:13.457049364Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:00:13.459203357Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T16:00:14.986380257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:16.415315091Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:00:18.425082163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:00:18.430378996Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:00:18.971024928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="549 B" avg_comp="549 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:00:18.97796413Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:00:19.054094159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:00:19.063618112Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:00:19.156878224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:00:19.163582784Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:00:19.637681824Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-07T16:00:19.637714084Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=warn ts=2026-05-07T16:00:19.98738955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:00:20.380356824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:00:22.844383153Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-07T16:00:24.9859908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:25.06829078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:00:25.072707554Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:00:26.057791855Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T16:00:26.953600657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58334"
level=info ts=2026-05-07T16:00:26.965400447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:00:26.97551074Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T16:00:27.079754054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:00:27.082978351Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:00:27.493145905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T16:00:27.498076669Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T16:00:27.609792296Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:00:29.272566539Z caller=checkpoint.go:372 msg="writing series" size="520 kB"
level=warn ts=2026-05-07T16:00:29.985759887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:00:30.223043102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:00:30.227150495Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:00:32.486969267Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-07T16:00:32.570221767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:00:32.57372772Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:00:34.955317229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:00:34.958831835Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:00:34.985719036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:00:35.283304377Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57672"
level=debug ts=2026-05-07T16:00:35.701236381Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-07T16:00:38.91507108Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-07T16:00:39.986117327Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:00:39.98612052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:39.986144783Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:00:39.986153017Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:00:40.405449891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:00:40.409405851Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:00:42.129635567Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-07T16:00:44.985719716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:45.345047603Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:00:45.345071111Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=8
level=info ts=2026-05-07T16:00:47.402195808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:00:47.405396674Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:00:48.558075751Z caller=checkpoint.go:372 msg="writing series" size="299 kB"
level=debug ts=2026-05-07T16:00:48.571105682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:00:49.986177207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:00:50.382661245Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:00:51.427215054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:00:51.4311198Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:00:51.772306196Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:00:54.850828712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:00:54.858444433Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:00:54.985651767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:00:54.986781334Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=debug ts=2026-05-07T16:00:55.29497449Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:00:57.882469958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:00:57.888601364Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:00:58.201049134Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
ts=2026-05-07T16:00:59.262093938Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58090"
level=warn ts=2026-05-07T16:00:59.986374131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:01.415523999Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-07T16:01:04.072942556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:01:04.079574998Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:01:04.079598323Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:01:04.079622641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:01:04.085571267Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:01:04.629233241Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-07T16:01:04.985791913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:07.9138076Z caller=checkpoint.go:372 msg="writing series" size="61 MB"
level=debug ts=2026-05-07T16:01:07.913841338Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=7
level=info ts=2026-05-07T16:01:08.984382436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:01:08.988308886Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:01:09.889876396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:01:09.898566776Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:01:09.966229427Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:01:09.966265824Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:01:09.981174084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:01:09.985529363Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:01:09.985531568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:09.985556518Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:01:09.985567571Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-05-07T16:01:09.988044346Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:01:09.988061214Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:01:09.988085017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:01:09.99547795Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:01:11.058479495Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-07T16:01:14.272848449Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-07T16:01:14.272873397Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-05-07T16:01:14.408709958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:01:14.415008947Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:01:14.985969435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:15.54118979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:01:15.544686279Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:01:17.201637581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:01:17.204566139Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:01:17.486323403Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:01:18.541495409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:01:18.544718178Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:01:19.986233435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:01:20.385159313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:01:20.700772742Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-05-07T16:01:20.712698997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="545 B" avg_comp="545 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:01:20.715856114Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:01:21.286449451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:01:21.289657199Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:01:22.180267308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:01:22.18744088Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T16:01:22.419137547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T16:01:22.42538237Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-07T16:01:23.914991978Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-07T16:01:24.725309968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:01:24.728736813Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:01:24.986000826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:26.650735677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:01:26.658217012Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:01:27.12968055Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
ts=2026-05-07T16:01:29.263527964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59976"
level=warn ts=2026-05-07T16:01:29.985860121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:30.191352797Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:01:30.344055161Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-07T16:01:33.558159619Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-07T16:01:34.669290846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:01:34.674833236Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:01:34.985689141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:01:35.289878337Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38920"
level=debug ts=2026-05-07T16:01:36.772653677Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-07T16:01:39.986386512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:39.986384689Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:01:39.986423554Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:01:39.986431599Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:01:39.986481093Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-07T16:01:40.617664098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:01:41.747521964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:01:41.757040558Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:01:41.757066881Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:01:41.757093933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:01:41.763403325Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:01:43.200714937Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-07T16:01:44.986110377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:44.998205656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:01:45.001722856Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:01:46.080820278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:01:46.087140528Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:01:46.087166179Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:01:46.087193076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:01:46.092690224Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:01:46.415643124Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=debug ts=2026-05-07T16:01:49.629306762Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T16:01:49.986301061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:01:50.353220314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:01:50.362760846Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:01:50.386889353Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T16:01:50.736895921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:01:50.740809932Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:01:50.864318456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:01:50.871296604Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:01:52.845820178Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-07T16:01:52.845843889Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=12
level=warn ts=2026-05-07T16:01:54.986080402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:01:56.05819385Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-07T16:01:58.397700898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:01:58.406157579Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:01:58.762991542Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:01:59.272562678Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=info ts=2026-05-07T16:01:59.701673417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:01:59.705765265Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:01:59.986469535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:02.486070824Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-07T16:02:03.121867736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:02:03.125239088Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:02:03.671837273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:02:03.677989515Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:02:04.986442822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:05.701199513Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=info ts=2026-05-07T16:02:07.718496203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:02:07.722673248Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:02:08.580396455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:02:08.588815067Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:02:08.717096971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:02:08.721199926Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:02:08.915292805Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:02:08.9483743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T16:02:08.953320102Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-07T16:02:09.966200619Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:02:09.96677104Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:02:09.986167604Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:02:09.986172162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:09.986195578Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:02:09.986204278Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:02:10.427906854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:02:10.431410991Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:02:12.12997763Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-05-07T16:02:14.985833732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:15.343598911Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=info ts=2026-05-07T16:02:16.509486098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:02:16.512807959Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:02:18.558005979Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=debug ts=2026-05-07T16:02:18.558032158Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-07T16:02:19.2619484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:02:19.265328275Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:02:19.986150929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:02:20.388946274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:02:20.395427058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:02:20.404345571Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T16:02:21.772771251Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=info ts=2026-05-07T16:02:24.628814132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:02:24.637379682Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:02:24.986451102Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-07T16:02:24.986460922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:26.271380217Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:02:26.963567556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:57164"
level=debug ts=2026-05-07T16:02:28.201329439Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-07T16:02:29.986513468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:30.839882575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:02:30.844195311Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:02:30.84421918Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:02:30.844242251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:02:30.84749405Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:02:31.415318587Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-07T16:02:34.630082773Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=warn ts=2026-05-07T16:02:34.985541814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:36.030381121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:02:36.03712043Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:02:36.37795581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:02:36.381335488Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:02:36.734125342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:02:36.740018702Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:02:36.911219325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:02:37.843677463Z caller=checkpoint.go:372 msg="writing series" size="281 kB"
level=info ts=2026-05-07T16:02:38.701972657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:02:38.708334177Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:02:39.985915238Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:02:39.985918549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:39.985941105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:02:39.98594975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:02:41.058439014Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T16:02:41.05846227Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=7
level=info ts=2026-05-07T16:02:43.65349708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:02:43.657690154Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:02:44.27195137Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-07T16:02:44.986295155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:02:47.486576476Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-05-07T16:02:48.264314344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:02:48.26939251Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:02:48.269414877Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:02:48.269437458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:02:48.272083189Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:02:48.927772353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:02:48.931193735Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:02:48.931214931Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:02:48.931235985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:02:48.934231901Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:02:49.173100587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:02:49.176598121Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:02:49.176620259Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:02:49.231819602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:02:49.235638278Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:02:49.62175823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T16:02:49.626329432Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T16:02:49.986179975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:50.127491707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:02:50.130792764Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:02:50.391367811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:02:50.700999281Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=debug ts=2026-05-07T16:02:53.915420479Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=warn ts=2026-05-07T16:02:54.986161107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:02:56.761301878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:02:56.764744999Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-07T16:02:56.965310548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49974"
level=debug ts=2026-05-07T16:02:57.129442296Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-07T16:02:59.986442529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:00.343624175Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-07T16:03:01.980582568Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:03:03.55778382Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T16:03:03.698273138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:03:03.701943818Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:03:04.087290004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:03:04.09103327Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:03:04.266552624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:03:04.27386546Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:03:04.986037872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:06.772437187Z caller=checkpoint.go:372 msg="writing series" size="501 kB"
level=debug ts=2026-05-07T16:03:06.772460908Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-05-07T16:03:07.859296795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:03:07.86277934Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-07T16:03:07.862802707Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:03:07.862823398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:03:07.865378877Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:03:08.666650143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:03:08.669787307Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:03:09.076222277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:03:09.079602346Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:03:09.966279968Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:03:09.967524674Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T16:03:09.985573471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:09.98558414Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:03:09.985601469Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:03:09.985609217Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:03:09.986738972Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-07T16:03:11.719991033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:03:11.727974364Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T16:03:11.895553788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:03:11.899374847Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:03:12.695746074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:03:12.699103342Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:03:12.699122955Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:03:12.699144123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:03:12.701730709Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:03:13.201321969Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T16:03:14.986406447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:15.9874014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:03:15.994460953Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:03:16.415657326Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:03:18.164427555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:03:18.167681768Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:03:19.630084163Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=warn ts=2026-05-07T16:03:19.985890117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:03:20.394003747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:03:22.843990552Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-07T16:03:22.844013838Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-07T16:03:23.499754926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:03:23.506087642Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:03:24.98599058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:26.058076018Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-07T16:03:27.328205905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:03:27.331615549Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:03:29.272968469Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T16:03:29.272990503Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
ts=2026-05-07T16:03:29.274122838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35216"
level=info ts=2026-05-07T16:03:29.406163168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:03:29.409511633Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:03:29.40953338Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:03:29.409552504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:03:29.412341478Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:03:29.986148173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:30.122543014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:03:30.128443678Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:03:32.432742494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:03:32.436100748Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:03:32.48631681Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-07T16:03:32.596677503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:03:32.600560149Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:03:34.532482695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:03:34.696067728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T16:03:34.700076135Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-07T16:03:34.985530377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:35.396156331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:03:35.399038354Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:03:35.701039373Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-05-07T16:03:35.826300279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:03:35.831957143Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:03:36.438315551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:03:38.915759543Z caller=checkpoint.go:372 msg="writing series" size="475 kB"
level=info ts=2026-05-07T16:03:39.73992718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:03:39.743578793Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:03:39.986102274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:03:39.986133233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:03:39.98615374Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:03:39.986162589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:03:41.671231707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:03:41.674404005Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:03:41.975482822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:03:41.978620279Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:03:42.129908953Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-07T16:03:44.986263272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:45.34368661Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-07T16:03:45.343717208Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-07T16:03:46.172099781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:03:46.179436099Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:03:48.286693779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:03:48.290413507Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:03:48.290436305Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:03:48.290459057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:03:48.293221427Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:03:48.558121163Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-07T16:03:49.218894915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:03:49.22230324Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:03:49.222322918Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:03:49.22234361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:03:49.224822299Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:03:49.985773256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:03:50.396565628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:03:51.774243845Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-07T16:03:51.774289787Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=2
level=warn ts=2026-05-07T16:03:54.986305355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:03:54.988000677Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T16:03:54.988015589Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=debug ts=2026-05-07T16:03:58.20139261Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=warn ts=2026-05-07T16:03:59.985923745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:00.13630661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:04:00.146121641Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:04:00.469153963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:04:00.476842379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:04:01.415488989Z caller=checkpoint.go:372 msg="writing series" size="996 B"
level=debug ts=2026-05-07T16:04:02.45162576Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:04:04.234938943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:04:04.245136665Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T16:04:04.629741447Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-07T16:04:04.985908788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:04:05.304026517Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:54786"
level=info ts=2026-05-07T16:04:05.859990856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:04:05.864562013Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:04:06.098225333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:04:06.101431759Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:04:06.223756151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:04:06.227175005Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:04:07.843630488Z caller=checkpoint.go:372 msg="writing series" size="645 kB"
level=info ts=2026-05-07T16:04:09.966300772Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:04:09.966734096Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:04:09.986150975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:04:09.986173711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:04:09.986175346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:09.986183426Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:04:11.058919365Z caller=checkpoint.go:372 msg="writing series" size="546 kB"
level=debug ts=2026-05-07T16:04:11.058943645Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-07T16:04:12.308175086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:04:12.311402398Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:04:13.993392634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:04:13.997526902Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:04:14.272454386Z caller=checkpoint.go:372 msg="writing series" size="438 kB"
level=warn ts=2026-05-07T16:04:14.986061339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:16.069714839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:04:16.075695252Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:04:17.486637379Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-07T16:04:19.985816661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:04:20.397802117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:04:20.704811128Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-07T16:04:20.704835455Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=3
level=info ts=2026-05-07T16:04:22.955740775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:04:22.96139517Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:04:23.482326304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="847 kB" avg_comp="847 kB" total_uncomp="6.7 MB" avg_uncomp="6.7 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-07T16:04:23.486933484Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e02eff7a3%3A19e03102acd%3A7574e37e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-07T16:04:23.486957515Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:23.486981586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="757 kB" avg_comp="757 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:04:23.49090905Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:04:23.914671438Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:04:24.986326828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:25.575596841Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:04:25.577430399Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:04:25.915311282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:04:25.918499237Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-07T16:04:26.971267669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47720"
level=debug ts=2026-05-07T16:04:27.129493419Z caller=checkpoint.go:372 msg="writing series" size="770 kB"
level=info ts=2026-05-07T16:04:27.791150298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:04:27.79463633Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:04:28.784913362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:04:28.788699942Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:04:28.788722009Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:28.788745502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:04:28.791376032Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:04:29.277647383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43086"
level=warn ts=2026-05-07T16:04:29.986011327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:30.344295569Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T16:04:30.344322744Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-07T16:04:33.335848022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:04:33.3421101Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:04:33.557914142Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-07T16:04:34.214163558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:04:34.214218235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="198 B" avg_comp="198 B" total_uncomp="615 B" avg_uncomp="615 B" idle=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:04:34.217523862Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=warn ts=2026-05-07T16:04:34.986224141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:04:35.307143036Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:48928"
level=debug ts=2026-05-07T16:04:36.772238703Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:04:38.483209014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="549 B" avg_comp="549 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:04:38.486872693Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:04:39.985944048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:04:39.985948416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:04:39.985970074Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:04:39.98597861Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:04:39.985994856Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T16:04:39.990903121Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-07T16:04:39.992332597Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006356911s
level=info ts=2026-05-07T16:04:40.551103539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:04:40.559427169Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:04:40.55945576Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:04:40.559503021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:04:40.566307286Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:04:43.006628503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:04:43.011194176Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:04:44.985485382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:45.072915444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:04:45.077082956Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:04:45.384951707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:04:45.388058943Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:04:45.912486339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:04:45.915751353Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:04:47.266673959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:04:47.269891549Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:04:49.986508983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:50.224527131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:04:50.22858672Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T16:04:50.399425736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T16:04:52.180037459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:04:52.188147255Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:04:54.985670648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:04:57.512218079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:04:57.520474369Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:04:57.678931599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:04:57.685033027Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:04:59.278721505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:57146"
level=warn ts=2026-05-07T16:04:59.986450366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:01.310948603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:05:01.32147481Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=error ts=2026-05-07T16:05:01.32150184Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:05:01.321527833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:05:01.328589477Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T16:05:02.940844798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:05:02.944557666Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:05:04.526178821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:05:04.530641028Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:05:04.985964814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:07.596143097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:05:07.599284422Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:05:08.136476151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:05:08.139531153Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:05:08.865647768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:05:08.870344812Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:05:09.191390686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:05:09.446906234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:05:09.44976324Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:05:09.966353954Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:05:09.966360305Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:05:09.985855845Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:05:09.985862596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:09.985879594Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:05:09.985887303Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:05:09.985888665Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T16:05:09.986146285Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-07T16:05:09.986155507Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:05:09.986173495Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.508µs success=true
level=debug ts=2026-05-07T16:05:09.986477628Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-07T16:05:12.543204421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:05:12.546067771Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:05:13.206348016Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T16:05:13.206372043Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=debug ts=2026-05-07T16:05:13.360523327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:05:14.288294264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:05:14.294859558Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T16:05:14.986211072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:15.26570195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:05:15.271712163Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:05:16.415816985Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:05:19.628234294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:05:19.631853183Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:05:19.64164072Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-07T16:05:19.641661874Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=2
level=warn ts=2026-05-07T16:05:19.985750195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:05:20.401002183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:05:22.843831786Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-07T16:05:24.985952547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:26.058248259Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-07T16:05:29.272543251Z caller=checkpoint.go:372 msg="writing series" size="520 kB"
ts=2026-05-07T16:05:29.280146741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:59858"
level=info ts=2026-05-07T16:05:29.570561741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:05:29.57804782Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:05:29.986456684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:05:30.625757474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:05:30.62968234Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:05:31.748461606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:05:31.755041327Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:05:32.487325697Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-07T16:05:34.986485711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:05:35.313448961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40550"
level=debug ts=2026-05-07T16:05:35.701619421Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-07T16:05:38.384530945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:05:38.391115514Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:05:38.594925716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:05:38.598131969Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:05:38.916009883Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-07T16:05:39.985877935Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:05:39.985898458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:39.985907074Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:05:39.985921904Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:05:40.611330624Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:05:42.129350757Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-07T16:05:44.393171555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:05:44.397115729Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:05:44.986337297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:45.344940911Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:05:45.344962421Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=8
level=info ts=2026-05-07T16:05:45.946591225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:05:45.954930553Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T16:05:46.682181168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:05:46.685706468Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:05:47.764860447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:05:47.768536794Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:05:47.936927906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:05:47.941698101Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:05:48.558734288Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-07T16:05:49.986350178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:05:50.402460266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T16:05:51.30741969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:05:51.313586753Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:05:51.773127228Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:05:52.744083861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:05:52.747410292Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:05:53.565939107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:05:53.569075861Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:05:54.985592364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:05:54.986732775Z caller=checkpoint.go:372 msg="writing series" size="4.7 kB"
level=info ts=2026-05-07T16:05:55.357862556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:05:55.361345Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:05:55.508306501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:05:55.511583486Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:05:58.201442917Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=info ts=2026-05-07T16:05:59.240048356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:05:59.245161241Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-07T16:05:59.281591527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45190"
level=warn ts=2026-05-07T16:05:59.985820842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:01.415583662Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-07T16:06:04.630064001Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-07T16:06:04.726412531Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:06:04.986127338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:06:05.316894383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56452"
level=debug ts=2026-05-07T16:06:07.907283163Z caller=checkpoint.go:372 msg="writing series" size="61 MB"
level=debug ts=2026-05-07T16:06:07.907314423Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=7
level=debug ts=2026-05-07T16:06:08.024027241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:06:09.372789915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:06:09.378286214Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:06:09.966190412Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:06:09.966692348Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:06:09.985612482Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:06:09.985641456Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:06:09.985649506Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:06:09.985644288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:09.997885387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:06:10.00958575Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:06:11.05950385Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:06:11.05952817Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=debug ts=2026-05-07T16:06:14.27305371Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=warn ts=2026-05-07T16:06:14.986246274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:17.487143512Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:06:17.95968942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:06:17.962823836Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:06:18.217701895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:06:18.221363422Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:06:19.986361097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:06:20.405245891Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:06:20.701659143Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-07T16:06:23.109063447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:06:23.116010756Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:06:23.915408731Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-07T16:06:24.985725273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:27.129417896Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
ts=2026-05-07T16:06:29.282997664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37052"
level=info ts=2026-05-07T16:06:29.736516741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:06:29.741203745Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:06:29.98603232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:30.343932293Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=info ts=2026-05-07T16:06:30.917486245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:06:30.920769808Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:06:32.021148903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:06:32.021193662Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-07T16:06:32.025390975Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:06:33.558653978Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T16:06:33.558680478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-07T16:06:34.746703288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:06:34.750490405Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:06:34.985829724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:06:35.319870354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38526"
level=info ts=2026-05-07T16:06:35.477196913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:06:35.484698041Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:06:36.387667349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:06:36.391231892Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:06:36.645797538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:06:36.648944519Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:06:36.77248134Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-07T16:06:37.853396163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:06:37.858793981Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:06:39.606518732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:06:39.61249822Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:06:39.986214798Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:06:39.986240919Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:06:39.986249238Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:06:39.986244605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:39.987389496Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-07T16:06:40.225789337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:06:40.228927202Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:06:40.281162561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="545 B" avg_comp="545 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:06:40.283926058Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:06:42.274074179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:06:42.277370459Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:06:42.439722072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:06:42.447401777Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T16:06:43.200719735Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-07T16:06:44.986405063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:06:45.442982281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:06:45.449031643Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:06:46.415737665Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-07T16:06:49.016705994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:06:49.020674242Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:06:49.630448667Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T16:06:49.986475044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:06:50.407369543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T16:06:51.790849025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:06:51.798782819Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:06:52.84563164Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-07T16:06:52.845655538Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=6
level=warn ts=2026-05-07T16:06:54.986309406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:06:56.058892871Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-07T16:06:59.273176796Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=warn ts=2026-05-07T16:06:59.986298632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:02.487306306Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=debug ts=2026-05-07T16:07:02.505552243Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:07:04.986440316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:05.70179891Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=debug ts=2026-05-07T16:07:06.221413631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:07:08.332125196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:07:08.33874102Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:07:08.576708015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:07:08.579876274Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:07:08.915694975Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:07:09.959814969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:07:09.962781057Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:07:09.966420413Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:07:09.966434084Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:07:09.985969186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:07:09.985977197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:09.985989201Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:07:09.985997401Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:07:10.066532705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:07:10.069427101Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:07:12.130045289Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=warn ts=2026-05-07T16:07:14.985839271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:15.344822993Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=debug ts=2026-05-07T16:07:15.344848556Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-07T16:07:15.858964125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:07:15.863431616Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:07:17.701309784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:07:17.704541475Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:07:18.425730538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:07:18.42916585Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:07:18.558665459Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=info ts=2026-05-07T16:07:19.157097184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:07:19.161621024Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:07:19.986075775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:07:20.409189074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:07:21.772949292Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-07T16:07:22.139725991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:07:22.143314693Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:07:24.986090567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:24.987227357Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-07T16:07:25.258132687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:07:25.261817653Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:07:25.630148382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:07:25.634167548Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:07:26.101695104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:07:26.108397933Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:07:27.00802344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:07:27.012635762Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:07:28.201705667Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=debug ts=2026-05-07T16:07:28.476590514Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:07:29.985864316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:31.415965602Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-07T16:07:34.625791864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=debug ts=2026-05-07T16:07:34.630371575Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=error ts=2026-05-07T16:07:34.634415187Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T16:07:34.986191666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:07:35.902988234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:07:35.909717176Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:07:35.90974218Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:07:35.909771905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:07:35.915165064Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:07:36.025392497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:07:36.028659946Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:07:37.844467627Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=info ts=2026-05-07T16:07:39.460433749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:07:39.464527064Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:07:39.985896531Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:07:39.985899031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:39.985919864Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:07:39.985928892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:07:41.059934723Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T16:07:41.059961292Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=8
level=info ts=2026-05-07T16:07:42.079726519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:07:42.085633487Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:07:44.273186526Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-07T16:07:44.98590084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:07:47.374286138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:07:47.386398912Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:07:47.48677072Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=info ts=2026-05-07T16:07:48.572714793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:07:48.579694715Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:07:49.985790519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:07:50.410110824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:07:50.700980365Z caller=checkpoint.go:372 msg="writing series" size="7.6 kB"
level=debug ts=2026-05-07T16:07:52.434782296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:07:53.916344308Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=info ts=2026-05-07T16:07:53.971360838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:07:53.974599557Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:07:54.696194387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:07:54.699469381Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:07:54.986519739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:07:55.199353052Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:07:57.129628531Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-07T16:07:59.985539921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:00.344052861Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T16:08:02.179894528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:08:02.183474312Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:08:03.55828816Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-07T16:08:04.985502989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:05.328082914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:08:05.33510411Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:08:05.335125984Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:08:05.335152441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:08:05.341460133Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:08:06.772729994Z caller=checkpoint.go:372 msg="writing series" size="427 kB"
level=debug ts=2026-05-07T16:08:06.772752077Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-07T16:08:07.446795831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:08:07.450342339Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:08:09.96645375Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:08:09.966492461Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:08:09.98597624Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:08:09.986003039Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:08:09.98599766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:09.986012631Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:08:09.987170374Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-07T16:08:10.664649561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:08:10.668355423Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:08:11.917691374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:08:11.92094555Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:08:12.221924694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:08:12.225266521Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:08:12.921098429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:08:12.924805264Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:08:13.201728299Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-07T16:08:14.70721236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:08:14.713700434Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:08:14.986227231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:16.41573564Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:08:16.839122034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:08:16.847630414Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T16:08:19.035039178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:08:19.040795281Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:08:19.184379736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:08:19.190937091Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:08:19.631307501Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=warn ts=2026-05-07T16:08:19.985820538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:08:20.41104766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:08:22.844583102Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-07T16:08:22.844614002Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-07T16:08:23.669885931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:08:23.674709456Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:08:24.267304165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:08:24.271238815Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:08:24.487063423Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:08:24.985687955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:25.25530191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:08:25.262079706Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:08:25.262105898Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:08:25.262134579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:08:25.268010419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:08:25.435780358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:08:25.440769019Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:08:25.440791365Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:08:25.562186404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:08:25.567507592Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:08:26.05889097Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
ts=2026-05-07T16:08:26.987441576Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47624"
level=debug ts=2026-05-07T16:08:29.27295776Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T16:08:29.272981563Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=warn ts=2026-05-07T16:08:29.986175399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:30.809463187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:08:30.812869555Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:08:32.09642508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:08:32.099564094Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:08:32.486669436Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-07T16:08:34.986053509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:35.700973485Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-05-07T16:08:36.193278169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:08:36.196791252Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:08:38.915610196Z caller=checkpoint.go:372 msg="writing series" size="545 kB"
level=debug ts=2026-05-07T16:08:38.915633858Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-07T16:08:39.98617508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:08:39.986202103Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:08:39.986210147Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:08:39.986207357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:08:41.042497342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:08:41.050087002Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:08:41.368084006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:08:41.376188251Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:08:41.92857568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:08:41.934143097Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:08:42.129691739Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-07T16:08:44.986043661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:45.344448678Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-07T16:08:46.579067077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:08:46.58252131Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:08:47.182948259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:08:47.184728714Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:08:48.492108389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:08:48.498343511Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:08:48.55865054Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-07T16:08:48.794625278Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:08:49.985471135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:08:50.413302621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:08:51.77506402Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-07T16:08:51.775085849Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=4
level=info ts=2026-05-07T16:08:54.045378599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:08:54.049490712Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:08:54.90898355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:08:54.912539811Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:08:54.985989508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:08:54.98878273Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T16:08:54.988797809Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=info ts=2026-05-07T16:08:57.818190185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:08:57.824361145Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:08:58.201670427Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-07T16:08:59.985645092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:00.717282028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:09:00.72683898Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=debug ts=2026-05-07T16:09:01.416009445Z caller=checkpoint.go:372 msg="writing series" size="957 B"
level=info ts=2026-05-07T16:09:02.57040147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:09:02.573896072Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:09:03.23663647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:09:03.240105324Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:09:04.029347761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:09:04.032727345Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:09:04.032752815Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:09:04.032775569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:09:04.035354795Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:09:04.442631529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:09:04.445933253Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:09:04.629538298Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-07T16:09:04.986550468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:05.866296749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:09:05.869936238Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:09:05.874977111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:09:05.880932519Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:09:07.844248383Z caller=checkpoint.go:372 msg="writing series" size="645 kB"
level=info ts=2026-05-07T16:09:09.019308448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:09:09.024473874Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:09:09.966267694Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:09:09.967049433Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:09:09.985503166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:09:09.985504862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:09.985527765Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:09:09.985538696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:09:11.058979638Z caller=checkpoint.go:372 msg="writing series" size="557 kB"
level=debug ts=2026-05-07T16:09:11.059002771Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-07T16:09:12.046315214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:09:12.05196918Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:09:14.273007567Z caller=checkpoint.go:372 msg="writing series" size="573 kB"
level=warn ts=2026-05-07T16:09:14.985707075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:17.486636397Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-07T16:09:18.870143871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:09:18.873926076Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:09:19.986387776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:20.191320716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:09:20.195769738Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:09:20.195792156Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:09:20.19581857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:09:20.199311626Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T16:09:20.416418041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:09:20.706559165Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-05-07T16:09:20.70658681Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.7 MB" series=3
level=info ts=2026-05-07T16:09:22.428718266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:09:22.43201236Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:09:23.915610479Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-07T16:09:24.196725394Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:09:24.986181217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:27.130597689Z caller=checkpoint.go:372 msg="writing series" size="780 kB"
level=info ts=2026-05-07T16:09:29.334154521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:09:29.339179276Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:09:29.985999754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:30.344024464Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T16:09:30.344055509Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-07T16:09:30.791212047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="628 B" avg_comp="628 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:09:30.794302281Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:09:30.974941202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:09:30.97872623Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-07T16:09:30.978744637Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:09:30.978765825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:09:30.981152135Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:09:33.408964787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:09:33.415722856Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:09:33.558203263Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T16:09:34.985935926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:36.300744111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:09:36.772316661Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:09:38.63831221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:09:38.64485125Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:09:39.954647524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:09:39.95879362Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:09:39.986347613Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:09:39.986380417Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:09:39.986388556Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:09:39.986382791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:09:39.986385835Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T16:09:39.992323472Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-07T16:09:39.99357387Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007236852s
level=info ts=2026-05-07T16:09:43.831023195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:09:43.834712934Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:09:44.985644667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:45.943652943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:09:45.946646444Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:09:46.462429546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:09:46.465201279Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:09:46.465222245Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:09:46.465242954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:09:46.467375263Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:09:49.725921707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:09:49.729098991Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:09:49.985653765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:50.241235469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:09:50.246966452Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:09:50.418826046Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:09:51.573663278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:09:51.576609841Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:09:51.576629791Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:09:51.576651031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:09:51.579059653Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:09:54.985887205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:09:55.051031506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:09:55.054050496Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:09:55.392126733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:09:55.400046227Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=info ts=2026-05-07T16:09:56.171429242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:09:56.174547383Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:09:56.735097165Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-07T16:09:56.994100732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34968"
ts=2026-05-07T16:09:59.29751811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51858"
level=warn ts=2026-05-07T16:09:59.98597553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:10:00.095295396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:10:00.099307599Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:10:01.012644186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="549 B" avg_comp="549 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:10:01.015918914Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:10:01.015940024Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:10:01.015960825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:10:01.018793084Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:10:01.072103288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:10:01.0769867Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:10:01.244384203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:10:01.251545111Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:10:03.600710163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:10:03.603750889Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:10:04.986377778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:10:05.340437577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:60532"
level=info ts=2026-05-07T16:10:09.552029797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:10:09.557317971Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:10:09.966417303Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:10:09.966420327Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:10:09.985947144Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T16:10:09.985992751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:10:09.985990064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:10:09.986012943Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:10:09.986021174Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:10:09.986193742Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-07T16:10:09.986203581Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:10:09.986223245Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.178µs success=true
level=debug ts=2026-05-07T16:10:09.986534359Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-07T16:10:10.625914618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:10:10.629432561Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:10:10.737505654Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:10:13.205431761Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T16:10:13.205454583Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=info ts=2026-05-07T16:10:13.46561383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:10:13.468526472Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:10:13.608967706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:10:13.615547077Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:10:14.986003804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:16.415438426Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:10:17.751144481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:10:17.754073963Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:10:19.044629106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:10:19.047708084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:10:19.642076307Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-07T16:10:19.642101951Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=2
level=warn ts=2026-05-07T16:10:19.986393006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:10:20.420473477Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T16:10:21.589480041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:10:21.592653865Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:10:21.5926776Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:10:21.592705592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:10:21.595101131Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:10:22.844752548Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-07T16:10:24.710724416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:10:24.714443684Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:10:24.985979121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:26.058872226Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:10:26.564168327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:10:26.56755698Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:10:28.245392137Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:10:28.446909683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:10:28.450786748Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:10:29.272534697Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=warn ts=2026-05-07T16:10:29.985540111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:10:30.976903673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:10:30.981591903Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:10:30.981613617Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:10:30.981635757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:10:30.984331323Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:10:32.123250424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:10:32.12829184Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:10:32.487370469Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-07T16:10:33.676684281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:10:33.682361958Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:10:34.986535244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:10:35.695063805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:10:35.698168909Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:10:35.698190104Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:10:35.698211504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:10:35.700761293Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:10:35.700817244Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-07T16:10:36.747794498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:10:36.753693288Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:10:38.916052551Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-07T16:10:39.417465165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:10:39.420754108Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:10:39.420775084Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:10:39.42079749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:10:39.423174801Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:10:39.986435221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:10:39.986440101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:10:39.98646155Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:10:39.986470198Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:10:42.129369123Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-07T16:10:42.625904529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:10:42.629176396Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:10:42.734385501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:10:42.741652455Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:10:44.985969084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:45.168492471Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:10:45.345613441Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:10:45.34563932Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=8
level=info ts=2026-05-07T16:10:46.01153121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:10:46.014695298Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:10:47.480743501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:10:47.483724672Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:10:48.558608485Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=warn ts=2026-05-07T16:10:49.985996155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:10:50.421995813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:10:51.77245529Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:10:53.419023323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:10:53.422197924Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:10:53.457306285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:10:53.464324119Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=warn ts=2026-05-07T16:10:54.985605112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:10:54.986728504Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-07T16:10:56.449300887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:10:56.452753853Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:10:57.635187867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:10:57.638179082Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:10:58.20142803Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=info ts=2026-05-07T16:10:58.977540837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:10:58.98286145Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:10:59.641730748Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:10:59.986514397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:11:01.415706217Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-07T16:11:02.238113012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:11:02.245994876Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:11:02.246018239Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:02.246058462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:11:02.252959967Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:11:03.374368863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:11:03.378870703Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:11:03.378893031Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:03.378917861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:11:03.382357538Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:11:04.629637254Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-07T16:11:04.986301953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:05.040481159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:11:05.043882464Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-07T16:11:05.346647903Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:52390"
level=debug ts=2026-05-07T16:11:07.912449535Z caller=checkpoint.go:372 msg="writing series" size="61 MB"
level=debug ts=2026-05-07T16:11:07.912482077Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=7
level=info ts=2026-05-07T16:11:09.966320685Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:11:09.966809154Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:11:09.985909794Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:11:09.985932615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:11:09.985922705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:09.985940524Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:11:10.324030908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:11:10.327756664Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:11:11.059951566Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T16:11:11.059979653Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-07T16:11:12.697473468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:11:12.700703176Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:11:14.078761132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:11:14.081799485Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:11:14.273015878Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=warn ts=2026-05-07T16:11:14.986314698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:15.948701083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:11:15.954564486Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:11:16.813319158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:11:16.821630622Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:11:17.081469473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:11:17.088468101Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:11:17.340103979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:11:17.34341507Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:11:17.486983916Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:11:17.69764121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:11:17.700957481Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:11:17.700980941Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:17.70100805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:11:17.703491408Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=warn ts=2026-05-07T16:11:19.985929859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:11:20.423166199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:11:20.701501521Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-07T16:11:21.402445599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:11:21.405823347Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:11:22.308218529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:11:22.311865666Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:11:23.915676761Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-07T16:11:24.398248219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:11:24.404233364Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:11:24.574780059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:11:24.577811399Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:11:24.986151363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:11:26.999774828Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47782"
level=debug ts=2026-05-07T16:11:27.129809221Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-07T16:11:29.986397885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:11:30.343938365Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-07T16:11:30.607792283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:11:30.61167056Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:11:30.87346882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:11:30.876579007Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:11:32.379921482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:11:32.388123224Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:11:32.547455377Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:11:33.55874909Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T16:11:33.558774332Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-07T16:11:34.986460331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:11:36.772919032Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-07T16:11:39.986043023Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:11:39.986045469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:11:39.986067337Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:11:39.986074925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:11:39.987271374Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-07T16:11:40.547998191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:11:40.556321319Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:11:41.173131962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:11:41.18081384Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:11:42.075998473Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:11:43.201306313Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=info ts=2026-05-07T16:11:43.771910455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="545 B" avg_comp="545 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:11:43.775235683Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:11:44.506863011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=4cb6b706965bace5 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-leli\"}"
level=error ts=2026-05-07T16:11:44.514397472Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4cb6b706965bace5/19e02f897fb%3A19e0304f924%3Af0d3cfd5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-leli\"}"
level=error ts=2026-05-07T16:11:44.514423797Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:11:44.514449852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="368 B" avg_comp="368 B" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" idle=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:11:44.516899506Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=warn ts=2026-05-07T16:11:44.986005228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:11:46.415414345Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-07T16:11:46.529834547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:11:46.534282051Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:11:46.913313473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:11:46.917498896Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:11:49.630345935Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T16:11:49.985777767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:11:50.42432408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:11:52.847022487Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-07T16:11:52.847051249Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=6
level=warn ts=2026-05-07T16:11:54.986439253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:11:55.741924446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:11:55.745701239Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:11:56.058353141Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-07T16:11:59.272684031Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
ts=2026-05-07T16:11:59.304196045Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46492"
level=info ts=2026-05-07T16:11:59.79430812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:11:59.798007922Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=warn ts=2026-05-07T16:11:59.986470904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:00.106800487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:12:00.110638805Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:12:02.48674884Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-07T16:12:02.97637186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:12:02.980467311Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:12:03.072722889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:12:03.075876438Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:12:03.693671258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:12:04.986541881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:12:05.701576092Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=info ts=2026-05-07T16:12:06.730762623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:12:06.737723207Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:12:07.014630326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:12:07.018287652Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:12:07.442552317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:12:07.445745683Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:12:08.915345526Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:12:09.96618293Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:12:09.966753337Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:12:09.986177399Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:12:09.986197432Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:12:09.986205446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:12:09.986204904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:10.641587085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:12:10.645497335Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:12:12.129896402Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=info ts=2026-05-07T16:12:14.127082224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:12:14.130154276Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:12:14.986056896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:12:15.344470195Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=debug ts=2026-05-07T16:12:15.344495081Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-07T16:12:18.55902109Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-05-07T16:12:19.985867175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:20.097193768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:12:20.105533627Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-07T16:12:20.425644473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T16:12:21.70520467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:12:21.709151355Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:12:21.772624732Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=info ts=2026-05-07T16:12:22.498539006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:12:22.502129362Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:12:24.346036968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:12:24.352310997Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:12:24.749371248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:12:24.752658874Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:12:24.985795846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:12:24.986937217Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-05-07T16:12:27.004005937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:57002"
level=debug ts=2026-05-07T16:12:28.201286852Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-07T16:12:29.309318634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:12:29.317425146Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:12:29.472109918Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:12:29.985671765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:31.283631466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:12:31.286858802Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:12:31.415308344Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-07T16:12:34.511194376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:12:34.518657313Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:12:34.630280958Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=warn ts=2026-05-07T16:12:34.986494373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:12:35.355506439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:36586"
level=info ts=2026-05-07T16:12:35.923989558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:12:35.928680555Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:12:37.844273555Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-07T16:12:39.572181356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:12:39.574086517Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-07T16:12:39.98628397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:39.986310063Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:12:39.986332068Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:12:39.986343376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:12:41.058898786Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T16:12:41.058922253Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=8
level=info ts=2026-05-07T16:12:41.845209121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:12:41.849910727Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:12:43.796363564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:12:43.800487089Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:12:44.272913931Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-07T16:12:44.985914791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:45.704177193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:12:45.707842513Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:12:47.486941208Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-07T16:12:49.338932901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:12:49.342938093Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:12:49.927799515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:12:49.931905851Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:12:49.985549612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:12:50.427150151Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:12:50.700966176Z caller=checkpoint.go:372 msg="writing series" size="8.2 kB"
level=debug ts=2026-05-07T16:12:53.916035312Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=warn ts=2026-05-07T16:12:54.985833193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:12:55.076172197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:12:55.080296286Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:12:55.080319466Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:12:55.080342587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:12:55.083091789Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:12:56.135692489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:12:56.139231497Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:12:57.005518279Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38070"
level=debug ts=2026-05-07T16:12:57.130493765Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=debug ts=2026-05-07T16:12:57.890446158Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:12:59.985535936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:00.344381251Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T16:13:00.711086763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:13:00.71749967Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:13:03.558299162Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T16:13:03.723676911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:13:03.728035957Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:13:04.98647311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:13:05.358559532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56620"
level=info ts=2026-05-07T16:13:05.446633157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:13:05.450403135Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:13:06.298962701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:13:06.302986956Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:13:06.772859768Z caller=checkpoint.go:372 msg="writing series" size="475 kB"
level=debug ts=2026-05-07T16:13:06.772884238Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-05-07T16:13:09.966274273Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:13:09.966386713Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T16:13:09.985627362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:13:09.985631675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:13:09.985652891Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:13:09.985672557Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:13:09.986786824Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-07T16:13:13.158038905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:13:13.162155698Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:13:13.201329479Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T16:13:14.986124112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:13:16.251938086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:13:16.255208392Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:13:16.415720415Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:13:16.581230173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:13:16.588798684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:13:17.644698767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:13:17.651122352Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:13:18.141707029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:13:18.147629149Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:13:19.418303313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:13:19.425035734Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:13:19.601448384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:13:19.604927218Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=debug ts=2026-05-07T16:13:19.630616964Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=warn ts=2026-05-07T16:13:19.986162635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:13:20.428304153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:13:22.388283982Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:13:22.84409161Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-07T16:13:22.844115175Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-07T16:13:24.198980389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:13:24.203497536Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:13:24.986349374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:26.058750749Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-07T16:13:28.192247181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:13:28.199830662Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:13:29.273559974Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T16:13:29.273582678Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=warn ts=2026-05-07T16:13:29.985967748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:32.486849261Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-07T16:13:33.619172291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:13:33.622862688Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:13:34.985711865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:35.701114383Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-05-07T16:13:36.029632175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:13:36.033296308Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:13:36.972307199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:13:36.9752354Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:13:37.193381802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:13:38.91621028Z caller=checkpoint.go:372 msg="writing series" size="615 kB"
level=debug ts=2026-05-07T16:13:38.916237073Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-07T16:13:39.750288959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:13:39.75438899Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:13:39.986002951Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:13:39.986029686Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:13:39.986037405Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:13:39.986032713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:42.129610906Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-07T16:13:44.985899268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:45.344063483Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-07T16:13:45.430308615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:13:45.433640035Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:13:47.726924678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:13:47.733374078Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:13:48.558282527Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-07T16:13:49.169952982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:13:49.173868426Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:13:49.985599016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:13:50.428969545Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:13:51.774330834Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-07T16:13:51.774353875Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=4
level=debug ts=2026-05-07T16:13:54.019824525Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:13:54.885215101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:13:54.889549014Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:13:54.985484039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:13:54.988245737Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T16:13:54.988273737Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=info ts=2026-05-07T16:13:57.808100843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:13:57.811973613Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:13:58.201318743Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=info ts=2026-05-07T16:13:58.522992578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:13:58.529778783Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:13:59.985752323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:01.075746199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:14:01.079514607Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:14:01.415497707Z caller=checkpoint.go:372 msg="writing series" size="957 B"
level=info ts=2026-05-07T16:14:01.629109874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:14:01.632905339Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:14:04.630213827Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-07T16:14:04.986467823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:05.855039661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:14:05.859228986Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:14:07.844143598Z caller=checkpoint.go:372 msg="writing series" size="645 kB"
level=debug ts=2026-05-07T16:14:07.844168351Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-07T16:14:09.966210804Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:14:09.967496514Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:14:09.985814393Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:14:09.985837562Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:14:09.985847437Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:14:09.98585536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:10.746613776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:14:10.750128728Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=debug ts=2026-05-07T16:14:11.058562913Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
level=info ts=2026-05-07T16:14:11.166538763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:14:11.171061859Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:14:11.860568635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:14:11.866624926Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:14:11.866645778Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:14:11.866667708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:14:11.871376029Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:14:12.929216297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:14:12.934631776Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:14:13.910095393Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:14:14.161023195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:14:14.169092485Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:14:14.272724888Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=debug ts=2026-05-07T16:14:14.272746979Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-07T16:14:14.986519757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:15.719870707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:14:15.723019278Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:14:17.487234861Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-07T16:14:19.98564676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:14:20.430147283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:14:20.704400197Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-05-07T16:14:20.704423684Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=2
level=debug ts=2026-05-07T16:14:22.418581868Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:14:23.91575119Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:14:24.986492388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:26.306574279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:14:26.31554154Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:14:26.428937159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:14:26.43281926Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:14:27.130034399Z caller=checkpoint.go:372 msg="writing series" size="796 kB"
level=info ts=2026-05-07T16:14:27.713378639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:14:27.717092874Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:14:28.472997658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:14:28.477220509Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:14:28.815171101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:14:28.818449194Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:14:29.986285366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:14:30.344830913Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T16:14:30.34485416Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-07T16:14:31.787687571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="628 B" avg_comp="628 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:14:31.791085197Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:14:33.558058707Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T16:14:34.812654761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:14:34.815537525Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:14:34.98645817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:14:35.367665354Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:39456"
level=debug ts=2026-05-07T16:14:36.772670321Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:14:38.027631875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:14:38.035820043Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:14:38.035843743Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:14:38.035868456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:14:38.042470334Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:14:39.986057076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:39.986051468Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:14:39.986081308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:14:39.986091702Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:14:39.9871582Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T16:14:39.989867324Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-07T16:14:39.991132252Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004719127s
level=info ts=2026-05-07T16:14:44.872875779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:14:44.876341541Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:14:44.986336177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:45.306219789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:14:45.309741828Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:14:46.713985915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:14:46.720416314Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:14:49.388663122Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:14:49.388720488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="41 kB" avg_uncomp="41 kB" idle=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:14:49.392370466Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=warn ts=2026-05-07T16:14:49.986430628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:14:50.43268097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:14:53.16106544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:14:53.168144347Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:14:53.168174295Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:14:53.168204694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:14:53.174414944Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:14:53.399817745Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:14:53.746691295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:14:53.7496255Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:14:54.560031995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:14:54.56305811Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:14:54.98571831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:14:56.013651939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:14:56.019516451Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:14:57.579203088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:14:57.582247805Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:14:57.620303545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="549 B" avg_comp="549 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:14:57.623078112Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:14:59.986382497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:15:01.183774784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:15:01.187298655Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=info ts=2026-05-07T16:15:03.558161524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:15:03.562441317Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:15:04.167061796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:15:04.170445452Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:15:04.986267648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:15:05.370754646Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:41036"
level=info ts=2026-05-07T16:15:09.115767225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:15:09.121201107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:15:09.123243058Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-07T16:15:09.12324776Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:15:09.966210002Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:15:09.966332808Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=debug ts=2026-05-07T16:15:09.968681214Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T16:15:09.968719515Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975744 n=1
level=debug ts=2026-05-07T16:15:09.968776844Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975744 n=1
level=info ts=2026-05-07T16:15:09.98615202Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:15:09.986156509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:09.986171229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:15:09.986178938Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:15:09.986181725Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T16:15:09.986305754Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-07T16:15:09.9863122Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:15:09.986329243Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.325µs success=true
level=debug ts=2026-05-07T16:15:09.986599879Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-07T16:15:10.274280685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:15:10.277732046Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:15:10.570284454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:15:10.578341378Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:15:12.631152478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:15:12.636553359Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:15:12.636580322Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:15:12.636605007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:15:12.640922246Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:15:13.208324171Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T16:15:13.208346521Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=warn ts=2026-05-07T16:15:14.986561726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:16.415282878Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T16:15:19.578301482Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:15:19.58529088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:15:19.589461136Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:15:19.643155437Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-07T16:15:19.643180173Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=2
level=warn ts=2026-05-07T16:15:19.985561056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:15:20.43485501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T16:15:21.094420204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:15:21.097818438Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:15:22.843901635Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-07T16:15:24.985735575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:15:25.970657572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:15:25.9753003Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:15:26.058545108Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:15:27.757943539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:15:27.762410831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:15:28.858650441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:15:28.864903427Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:15:29.273175188Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
ts=2026-05-07T16:15:29.320849227Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40858"
level=warn ts=2026-05-07T16:15:29.986400448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:30.361846263Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:15:31.811900464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:15:31.817709617Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:15:32.486733728Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-07T16:15:34.986307882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:15:35.113662412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:15:35.117065579Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:15:35.701009549Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-07T16:15:37.113187372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:15:37.119681067Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:15:38.915868959Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-05-07T16:15:39.894774987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:15:39.898171145Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=info ts=2026-05-07T16:15:39.985920434Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:15:39.985946459Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:15:39.985941522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:15:39.98595419Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:15:40.710928291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:15:40.718688299Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:15:42.129613423Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-07T16:15:43.136499564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:15:43.139881596Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:15:43.139902565Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:15:43.139925194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:15:43.142275787Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:15:44.986240923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:45.345248187Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:15:45.34529543Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=8
level=debug ts=2026-05-07T16:15:45.396099044Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:15:46.029127304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:15:46.033551971Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:15:48.558916413Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-05-07T16:15:48.600109242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:15:48.603816059Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=warn ts=2026-05-07T16:15:49.98601355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:15:50.436530651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T16:15:51.170597572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:15:51.173848632Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:15:51.772609158Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:15:52.311121969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:15:52.3151113Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:15:53.386309227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:15:53.394229523Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:15:54.055015316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:15:54.060202111Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:15:54.438215327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:15:54.441572277Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:15:54.986190671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:15:54.987314468Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-07T16:15:55.000378623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:15:55.003629676Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:15:56.421769108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:15:56.425683351Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:15:56.425705013Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:15:56.425728229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:15:56.428382067Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-07T16:15:57.015865742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:50860"
level=debug ts=2026-05-07T16:15:58.201727739Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=warn ts=2026-05-07T16:15:59.985815028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:00.913696771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:16:00.919552603Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:16:01.416086033Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-07T16:16:04.630433606Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-07T16:16:04.986057495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:16:07.9088642Z caller=checkpoint.go:372 msg="writing series" size="61 MB"
level=debug ts=2026-05-07T16:16:07.908895044Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=7
level=info ts=2026-05-07T16:16:08.207950415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:16:08.211755143Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:16:08.313999964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:16:08.317649384Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-07T16:16:08.317672853Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:16:08.317696413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:16:08.320513676Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:16:09.96633047Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:16:09.966401714Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:16:09.985537207Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:16:09.985565253Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:16:09.985574999Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:16:09.985577634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:10.467892424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:16:10.470783323Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:16:11.05949323Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T16:16:11.059515807Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=debug ts=2026-05-07T16:16:11.71667256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:16:11.718177795Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:16:13.385607209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:16:13.389480758Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:16:13.389502711Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:16:13.389525882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:16:13.392627635Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:16:14.273793168Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=warn ts=2026-05-07T16:16:14.986486945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:17.217046101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:16:17.220830007Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:16:17.486649949Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:16:17.878947968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:16:17.882284572Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:16:18.092028982Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:16:19.985975137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:16:20.437944542Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:16:20.701560748Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-07T16:16:22.517703211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:16:22.520825113Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:16:23.915978702Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T16:16:24.986411569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:25.534745796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:16:25.542026598Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:16:27.129726906Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=info ts=2026-05-07T16:16:27.826695498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:16:27.831642548Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:16:27.905863607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:16:27.909633142Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:16:28.775357523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="545 B" avg_comp="545 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:16:28.778634038Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-07T16:16:29.323727258Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50832"
level=warn ts=2026-05-07T16:16:29.985672511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:30.055909325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:16:30.059868635Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:16:30.059892404Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:16:30.059914741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:16:30.062195111Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:16:30.344820144Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-07T16:16:32.209984917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:16:32.215799105Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:16:33.558783577Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T16:16:33.558805335Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-07T16:16:34.986266766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:16:35.032444104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:16:35.036217536Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:16:35.379550135Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45136"
level=info ts=2026-05-07T16:16:36.612306541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:16:36.620140515Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:16:36.772696481Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-07T16:16:37.236058131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:16:37.238970707Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:16:37.238991953Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:16:37.23901636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:16:37.241247873Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:16:37.818243771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:16:37.82112749Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:16:39.986003363Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:16:39.986039345Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:16:39.986049767Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:16:39.986050471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:16:39.987185618Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-07T16:16:40.524323518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:16:40.530002601Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:16:40.652432294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:16:40.659529599Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:16:41.655709872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:16:41.658918149Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:16:42.630531978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:16:42.635764134Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:16:43.20163176Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-07T16:16:43.7511503Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:16:44.985656121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:16:46.415377282Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-07T16:16:46.826454838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:16:46.830020064Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:16:49.629756753Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-07T16:16:49.857437831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:16:49.862278374Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:16:49.986074172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:16:50.43919825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T16:16:51.730310519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:16:51.733871371Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:16:52.09944658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:16:52.102763073Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:16:52.847017461Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-07T16:16:52.847041745Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=info ts=2026-05-07T16:16:53.086222883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:16:53.090065325Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:16:54.919995099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:16:54.923424812Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:16:54.986507366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:16:56.058439249Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
ts=2026-05-07T16:16:57.019382062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:43874"
level=debug ts=2026-05-07T16:16:59.27355735Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=warn ts=2026-05-07T16:16:59.986154859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:02.486941695Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=warn ts=2026-05-07T16:17:04.985821198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:05.702222246Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=info ts=2026-05-07T16:17:07.145885609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:17:07.151398198Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:17:08.915774661Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:17:09.920300462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:17:09.923526831Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=info ts=2026-05-07T16:17:09.966365217Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:17:09.966720623Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T16:17:09.985873744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:17:09.985894649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:17:09.985919772Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:17:09.985928449Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:17:12.129858833Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=info ts=2026-05-07T16:17:12.332483743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:17:12.334023294Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:17:12.335372012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:17:12.395728095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:17:12.398665266Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:17:12.398683935Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:12.398704757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:17:12.400997295Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:17:13.998098528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:17:14.001301298Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=warn ts=2026-05-07T16:17:14.985651569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:15.344362757Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=debug ts=2026-05-07T16:17:15.344385356Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-07T16:17:16.113538309Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:17:17.523565116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:17:17.530137475Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:17:18.558722086Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-05-07T16:17:19.985829224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:17:20.44134785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:17:21.772805836Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-07T16:17:22.619951387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:17:22.624174602Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:17:23.182731435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:17:23.190843186Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:17:23.190866299Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:23.190906199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:17:23.198329372Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:17:23.276500612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:17:23.283348523Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:17:23.92491558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:17:23.930801596Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:17:24.986291427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:24.987417Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-07T16:17:26.054092795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:17:26.057630527Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:17:26.057649665Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:26.057670975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:17:26.060212588Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:17:26.276907655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:17:26.280155745Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:17:26.280175976Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:26.358295604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:17:26.361513828Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:17:28.201321944Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-07T16:17:28.550302408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:17:28.558536595Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:17:29.611522317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:17:29.614770057Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:17:29.735021781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:17:29.739579289Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:17:29.739601516Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:29.739627007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:17:29.743526075Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:17:29.953299318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:17:29.956617496Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:17:29.985979706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:17:30.560746337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:17:30.56409396Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:17:31.415526391Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-07T16:17:33.488544831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:17:33.491850255Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=debug ts=2026-05-07T16:17:34.630343083Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-05-07T16:17:34.986013945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:17:37.640758746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:17:37.649845798Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:17:37.844516251Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-05-07T16:17:38.349770614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:17:38.352953929Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:17:38.469459368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:17:38.472776427Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:17:39.985581063Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:17:39.985607138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:17:39.985600814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:17:39.985615102Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:17:41.059090737Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T16:17:41.059114836Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=8
level=info ts=2026-05-07T16:17:41.785177372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:17:41.789134422Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:17:41.789160841Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:41.789184186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:17:41.792008891Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:17:43.463647074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:17:43.468419909Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:17:44.175569607Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:17:44.272857013Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-07T16:17:44.986484318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:17:47.487557333Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-07T16:17:47.805687485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:17:47.809033305Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:17:49.985503027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:17:50.443999689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:17:50.701712133Z caller=checkpoint.go:372 msg="writing series" size="8.2 kB"
level=info ts=2026-05-07T16:17:51.786690927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:17:51.790760784Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:17:51.955065122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:17:51.958536116Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:17:52.71331666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:17:53.785306245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:17:53.788567437Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:17:53.915968974Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=info ts=2026-05-07T16:17:54.683230688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:17:54.686038457Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:17:54.686057943Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:17:54.686076456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:17:54.688188114Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=warn ts=2026-05-07T16:17:54.985773708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:17:57.022932404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:44252"
level=debug ts=2026-05-07T16:17:57.130143415Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=info ts=2026-05-07T16:17:57.496123897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:17:57.499211342Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-07T16:17:59.328417078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:46778"
level=warn ts=2026-05-07T16:17:59.986380592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:00.344283638Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T16:18:01.548210271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:18:01.551416916Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:18:01.826230094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:18:01.829406707Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:18:03.558534341Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T16:18:04.751920566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:18:04.758318952Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=warn ts=2026-05-07T16:18:04.986237636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:18:05.387943853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47096"
level=info ts=2026-05-07T16:18:05.40253539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:18:05.405964252Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:18:05.641486321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:18:05.644645749Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:18:06.773589562Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-07T16:18:06.773611449Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=8
level=info ts=2026-05-07T16:18:09.966227468Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:18:09.967344083Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:18:09.985503873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:18:09.985527375Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:18:09.985538336Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:18:09.985537214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:09.986671226Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-07T16:18:10.682448554Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:18:10.912918788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:18:10.91703093Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:18:13.20133326Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-07T16:18:14.117623611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:18:14.124578222Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:18:14.985766795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:18:15.157236591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:18:15.161205977Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:18:15.306337043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:18:15.313469649Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:18:15.529057344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:18:15.532113547Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:18:16.415183601Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T16:18:19.630271323Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=warn ts=2026-05-07T16:18:19.986115672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:18:20.00437263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:18:20.010098392Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-07T16:18:20.447346293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:18:20.539901389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:18:20.544217155Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:18:20.791142705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:18:20.797390049Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:18:21.980788857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:18:21.983991977Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:18:22.844644234Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-07T16:18:22.844669502Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-07T16:18:24.985840185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:18:25.852814314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:18:25.860405216Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:18:26.058245832Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-07T16:18:26.258654954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:18:26.263096866Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:18:26.263118647Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:18:26.263141854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:18:26.26599969Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:18:28.467229734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:18:28.470726249Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:18:29.273582348Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T16:18:29.273607442Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=warn ts=2026-05-07T16:18:29.986051903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:32.487002096Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-07T16:18:34.35953073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:18:34.362972957Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:18:34.985772601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:18:35.390333905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45214"
level=debug ts=2026-05-07T16:18:35.701756482Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-05-07T16:18:36.294293354Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:18:38.116302913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:18:38.119651992Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:18:38.916138748Z caller=checkpoint.go:372 msg="writing series" size="484 kB"
level=info ts=2026-05-07T16:18:39.985924656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:18:39.98595433Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:18:39.985952175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:18:39.985964874Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:18:42.13021695Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-07T16:18:42.616705034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:18:42.619670973Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=warn ts=2026-05-07T16:18:44.985681906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:45.34469996Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-07T16:18:45.344726262Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-07T16:18:47.070652024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:18:47.075152501Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:18:47.551239439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:18:47.554484295Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:18:48.558306186Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-07T16:18:49.986389294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:18:50.450040969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:18:51.774756806Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-07T16:18:51.774778751Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=info ts=2026-05-07T16:18:51.817369983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:18:51.820675131Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:18:52.223733614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:18:52.226803676Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:18:52.662761818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:18:52.66580541Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:18:54.986535329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:18:54.987998066Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-07T16:18:54.988014126Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=debug ts=2026-05-07T16:18:54.994322762Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:18:56.967781509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:18:56.972641244Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:18:58.136955523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:18:58.140445875Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:18:58.201893861Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=info ts=2026-05-07T16:18:58.236922122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:18:58.24051333Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-07T16:18:59.330984123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37844"
level=warn ts=2026-05-07T16:18:59.986409326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:19:00.67180689Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:19:00.67186581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:19:00.676065737Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-07T16:19:01.415108718Z caller=checkpoint.go:372 msg="writing series" size="957 B"
level=info ts=2026-05-07T16:19:01.707873393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:19:01.71124968Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:19:02.700181361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:19:02.707356217Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:19:04.188646728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:19:04.195881016Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:19:04.327109515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:19:04.33350457Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:19:04.629435542Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-07T16:19:04.98651957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:07.425039426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:19:07.428835543Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:19:07.844497258Z caller=checkpoint.go:372 msg="writing series" size="645 kB"
level=info ts=2026-05-07T16:19:08.564308533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:19:08.569955986Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:19:08.860686239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:19:08.865009829Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:19:09.966219774Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:19:09.966539535Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T16:19:09.986243608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:09.986266459Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:19:09.986289888Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:19:09.986297592Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:19:11.058701135Z caller=checkpoint.go:372 msg="writing series" size="461 kB"
level=debug ts=2026-05-07T16:19:11.058725916Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-07T16:19:12.052293728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:19:12.056607813Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-07T16:19:14.27244146Z caller=checkpoint.go:372 msg="writing series" size="442 kB"
level=warn ts=2026-05-07T16:19:14.985908186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:15.68175442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:19:15.685205581Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=debug ts=2026-05-07T16:19:17.487208556Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-07T16:19:19.986186089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:19:20.452583683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:19:20.705420985Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-07T16:19:20.705444399Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=3
level=info ts=2026-05-07T16:19:22.496003857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:19:22.499290348Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:19:23.915871626Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:19:24.985693643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:25.016903109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:19:25.024342218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:19:26.77823689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="628 B" avg_comp="628 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:19:26.781738934Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
ts=2026-05-07T16:19:27.02890965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46652"
level=debug ts=2026-05-07T16:19:27.129909Z caller=checkpoint.go:372 msg="writing series" size="807 kB"
level=info ts=2026-05-07T16:19:28.631538309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:19:28.635723689Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
ts=2026-05-07T16:19:29.332881228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50394"
level=warn ts=2026-05-07T16:19:29.985994991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:19:30.344343729Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T16:19:30.344368463Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-07T16:19:30.583674016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:19:30.587465491Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:19:30.589633393Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:19:33.559027265Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-07T16:19:34.986415704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:36.117941424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:19:36.121556274Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:19:36.772978946Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=debug ts=2026-05-07T16:19:39.228137902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:19:39.98547328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:39.985485316Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:19:39.985509313Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:19:39.985517535Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:19:39.986563593Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T16:19:39.997134742Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-07T16:19:39.998446376Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011982269s
level=info ts=2026-05-07T16:19:41.54185738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:19:41.545246008Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:19:44.225272215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:19:44.230343922Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-07T16:19:44.986072167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:46.929850109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:19:46.933208442Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:19:47.469802013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:19:47.473039894Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:19:49.04211287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:19:49.047806015Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:19:49.199364565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="549 B" avg_comp="549 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:19:49.202677822Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:19:49.985804197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:19:50.453746938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T16:19:51.593922989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:19:51.598164292Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:19:51.598186577Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:19:51.598211452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:19:51.60116559Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:19:52.717308569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:19:52.721012413Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:19:53.166226508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:19:53.173157135Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:19:53.409699662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:19:53.413058676Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:19:54.985892024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:19:55.975172338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:19:55.979705583Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:19:56.701680614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:19:56.707989114Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:19:57.030148964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58140"
level=info ts=2026-05-07T16:19:58.405065971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:19:58.413425792Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:19:59.256671462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:19:59.261146574Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:19:59.6751906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:19:59.677952563Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=warn ts=2026-05-07T16:19:59.985653541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:01.214745405Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:20:04.986065681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:20:05.39992194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56094"
level=info ts=2026-05-07T16:20:09.966734005Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:20:09.967527936Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:20:09.985824673Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-07T16:20:09.985834013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:20:09.985856873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:20:09.985877886Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:20:09.98588692Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:20:09.985940031Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-07T16:20:09.985947642Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:20:09.985965977Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.687µs success=true
level=debug ts=2026-05-07T16:20:09.9862218Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-07T16:20:10.283691493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:20:10.28893283Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:20:13.208415505Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T16:20:13.208444366Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=warn ts=2026-05-07T16:20:14.985830577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:20:15.448108979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:20:15.455869154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:20:16.415426561Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:20:17.559698234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:20:17.563169547Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:20:19.642274295Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-07T16:20:19.642297918Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=warn ts=2026-05-07T16:20:19.985874746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:20:20.45602969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:20:22.844087716Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-07T16:20:24.986012614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:20:25.402381018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=info ts=2026-05-07T16:20:25.40653517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:20:25.406966821Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:20:25.409249059Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:20:25.482215248Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:20:26.058487158Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:20:26.795625678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:20:26.802408071Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T16:20:27.031532911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47050"
level=info ts=2026-05-07T16:20:27.484539672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:20:27.4881133Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:20:29.272317092Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
ts=2026-05-07T16:20:29.337728574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51424"
level=warn ts=2026-05-07T16:20:29.986155575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:20:31.482479404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:20:31.488101717Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:20:32.486713308Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-07T16:20:32.490771835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:20:32.494407888Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:20:32.895528619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:20:32.899346515Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:20:33.524167009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:20:33.527455768Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:20:33.565207418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:20:33.568174005Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:20:33.884407847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:20:33.8880913Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:20:34.985704318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:20:35.403029476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:39250"
level=debug ts=2026-05-07T16:20:35.638913967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:20:35.640838792Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-07T16:20:35.701130033Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-07T16:20:37.60329613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:20:37.606811209Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:20:38.915060886Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-07T16:20:39.986300864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-07T16:20:39.98630204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:39.986324206Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:20:39.986332509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:20:42.129751454Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-07T16:20:42.623133047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:20:42.627933287Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:20:42.767296381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:20:42.774126932Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:20:43.591228686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:20:43.594281709Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:20:44.98621867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:20:45.091521493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:20:45.094504163Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:20:45.345074501Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:20:45.345097586Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=8
level=info ts=2026-05-07T16:20:46.994022872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:20:46.997773206Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:20:47.371356072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:20:47.374649659Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:20:48.558711128Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=warn ts=2026-05-07T16:20:49.98639462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:20:50.458146821Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:20:51.772197407Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:20:51.983961503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:20:51.992183008Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:20:53.070297815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:20:53.076813314Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:20:53.076837396Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:20:53.076862883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:20:53.082185682Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:20:54.230526284Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:20:54.986286308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:20:54.986342414Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-05-07T16:20:58.201523546Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=warn ts=2026-05-07T16:20:59.986435834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:01.415637327Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-07T16:21:04.629218276Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-07T16:21:04.986080068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:07.153568357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:21:07.164220572Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:21:07.692675181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:21:07.698677836Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:21:07.911209477Z caller=checkpoint.go:372 msg="writing series" size="61 MB"
level=debug ts=2026-05-07T16:21:07.911244565Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=7
level=info ts=2026-05-07T16:21:09.902244147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:21:09.905800302Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:21:09.966204308Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:21:09.967162086Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:21:09.985520368Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:21:09.985545805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:21:09.985554662Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:21:09.985575278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:10.398062613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:21:10.402537165Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-07T16:21:11.059703751Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-07T16:21:11.059726992Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=debug ts=2026-05-07T16:21:14.272554028Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=warn ts=2026-05-07T16:21:14.986514611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:16.267862378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:21:16.271411562Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:21:17.179979284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:21:17.184564819Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:21:17.486313586Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-05-07T16:21:19.986027064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:20.322071917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:21:20.325571617Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-07T16:21:20.459601358Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:21:20.701469256Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-07T16:21:21.171002373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:21:22.362281821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:21:22.365753278Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:21:23.774938727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:21:23.778117175Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:21:23.778138168Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:21:23.778160482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:21:23.780532855Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:21:23.914857395Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-07T16:21:24.985954281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:25.211021355Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:21:25.46032016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:21:25.46561039Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:21:25.465632131Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:21:25.465653918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:21:25.470233334Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:21:27.130013653Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=info ts=2026-05-07T16:21:27.868457663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:21:27.871840154Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:21:29.291946613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:21:29.294913036Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-07T16:21:29.339685523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37726"
level=info ts=2026-05-07T16:21:29.475782447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:21:29.480203463Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:21:29.986313211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:30.344120136Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-07T16:21:30.467045054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:21:30.470245195Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:21:30.683656703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:21:30.686675959Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:21:30.879203652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:21:30.885600127Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:21:30.885626124Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:21:30.885654204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:21:30.891565837Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:21:31.757031381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:21:31.760007424Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:21:31.993687463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:21:31.996551628Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:21:33.557830919Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T16:21:33.557855911Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-07T16:21:34.985697791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:21:35.441970165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:21:35.451237476Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:21:36.221142605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:21:36.224360921Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=info ts=2026-05-07T16:21:36.694628619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:21:36.700549765Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:21:36.771883344Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-07T16:21:38.249419286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="545 B" avg_comp="545 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:21:38.255650143Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:21:39.986056369Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:21:39.986085419Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:21:39.986094607Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:21:39.986128847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:39.986211345Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-07T16:21:41.776154469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:21:41.779600093Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:21:43.200723728Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-07T16:21:43.615861727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:21:43.621953236Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:21:44.795124666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:21:44.802398922Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:21:44.985496413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:46.414770175Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=debug ts=2026-05-07T16:21:48.027432351Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:21:49.629878013Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T16:21:49.986035726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:21:50.461142126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T16:21:51.75029343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:21:51.75549944Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-07T16:21:52.846357307Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-07T16:21:52.846384635Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=6
level=info ts=2026-05-07T16:21:53.914845434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:21:53.922578104Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:21:53.922603787Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:21:53.922634871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:21:53.929507153Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:21:54.798148679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:21:54.801610889Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:21:54.986379669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:21:56.058005627Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-07T16:21:57.557117276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:21:57.560659879Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:21:59.272769366Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-07T16:21:59.986377927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:02.486495083Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-07T16:22:03.084419048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:22:03.090719609Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:22:04.486882144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:22:04.494473607Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:22:04.986234266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:05.70178742Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=info ts=2026-05-07T16:22:05.853953375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:22:05.858564345Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:22:08.468460492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:22:08.473305368Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:22:08.915531302Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:22:09.681891253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:22:09.685281164Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:22:09.966219421Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:22:09.966227888Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=warn ts=2026-05-07T16:22:09.986290883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:09.986316224Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:22:09.986337289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:22:09.986359485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:22:10.778451165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:22:10.781527369Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:22:12.129424352Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=warn ts=2026-05-07T16:22:14.985958947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:15.053099455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:22:15.056404724Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:22:15.056424764Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:22:15.056446883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:22:15.058979448Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:22:15.344517584Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=debug ts=2026-05-07T16:22:15.344542616Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-07T16:22:15.98363938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:22:15.987108924Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:22:16.603393478Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:22:17.523920166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:22:17.530630835Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:22:18.558128954Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=debug ts=2026-05-07T16:22:19.466383976Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:22:19.492367464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:22:19.495900935Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:22:19.985485755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:22:20.463884899Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:22:21.77244996Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-07T16:22:21.866476234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:22:21.87408789Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:22:24.037418673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:22:24.040707886Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:22:24.119938906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:22:24.123020579Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:22:24.123039961Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:22:24.123061939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:22:24.125542885Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:22:24.985849649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:24.986989725Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-07T16:22:25.046094932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:22:25.04944487Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:22:26.116915983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:22:26.120289009Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:22:28.201422516Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-07T16:22:28.349738874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:22:28.35421368Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:22:29.985989701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:30.719834112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:22:30.727048163Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:22:31.415315767Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-07T16:22:32.419174539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:22:32.422459982Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:22:32.982274739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:22:32.986553757Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-07T16:22:34.434755695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:22:34.440417253Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:22:34.630074027Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-07T16:22:34.985986593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:35.138318786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:22:35.142379648Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:22:37.844190247Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-07T16:22:39.9862904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:39.9863096Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:22:39.986329938Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:22:39.986338712Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:22:40.942048173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:22:40.947372529Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-07T16:22:40.947394384Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:22:40.947416384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:22:40.950019813Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:22:40.996382435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:22:40.999629875Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:22:40.999652249Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:22:41.002616818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:22:41.005459486Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:22:41.058586163Z caller=checkpoint.go:372 msg="writing series" size="986 kB"
level=debug ts=2026-05-07T16:22:41.058613792Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=8
level=info ts=2026-05-07T16:22:41.248134712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:22:41.251629196Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:22:44.000492036Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:22:44.272303008Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-07T16:22:44.9860191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:22:47.487134685Z caller=checkpoint.go:372 msg="writing series" size="268 kB"
level=info ts=2026-05-07T16:22:49.902863295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:22:49.909174235Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:22:49.986330128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:22:50.466429559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:22:50.700737269Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=info ts=2026-05-07T16:22:51.314486801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:22:51.318358017Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:22:53.558076536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:22:53.562527498Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:22:53.865495765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:22:53.868959332Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:22:53.915275748Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=info ts=2026-05-07T16:22:54.382604561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:22:54.387270093Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:22:54.986395106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:22:55.942009407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:22:55.945713562Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:22:57.129543509Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=info ts=2026-05-07T16:22:59.255064438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:22:59.258403918Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-07T16:22:59.343728288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:38450"
level=warn ts=2026-05-07T16:22:59.985511637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:00.344063415Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T16:23:00.479715668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:23:00.483324549Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:23:00.483354148Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:00.48337633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:23:00.486386679Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:23:03.435735701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:23:03.438922712Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:23:03.438943711Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:03.438965301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:23:03.441468959Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:23:03.557971055Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-07T16:23:04.986481355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:23:05.572673769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:23:05.575760192Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:23:06.264518492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:23:06.773407Z caller=checkpoint.go:372 msg="writing series" size="540 kB"
level=debug ts=2026-05-07T16:23:06.773430183Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=8
level=info ts=2026-05-07T16:23:06.846070733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:23:06.849423713Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:23:09.966467328Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:23:09.966947006Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:23:09.986378593Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:23:09.986400179Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:23:09.986408105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:23:09.98641719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:09.986516641Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-07T16:23:10.229385293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:23:10.232803377Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:23:11.017414377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:23:11.024455449Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:23:11.430646632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:23:11.43867463Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:23:13.200951426Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-07T16:23:13.216059406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:23:13.219069821Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:23:14.515378637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:23:14.520793811Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:23:14.986049297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:23:15.527249846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:23:15.530733897Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-07T16:23:15.644161203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:23:15.647222466Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=debug ts=2026-05-07T16:23:16.415337862Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:23:17.332899898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:23:17.336784561Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:23:18.218303631Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:23:19.629666864Z caller=checkpoint.go:372 msg="writing series" size="764 kB"
level=warn ts=2026-05-07T16:23:19.985997239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:23:20.468401116Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:23:21.104322774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:23:21.107859047Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:23:21.593426295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:23:21.596713378Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:23:22.84439548Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-07T16:23:22.844416525Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-07T16:23:24.380724517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:23:24.38581425Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:23:24.98652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:26.057872516Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=info ts=2026-05-07T16:23:27.273219711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:23:27.280270453Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:23:29.272749878Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T16:23:29.272773077Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
ts=2026-05-07T16:23:29.345393575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42028"
level=warn ts=2026-05-07T16:23:29.985844581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:32.486807832Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-07T16:23:33.830539725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:23:33.834115908Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:23:33.834139167Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:33.834162624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:23:33.837016395Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:23:34.986336947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:23:35.529174059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:23:35.532385911Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:23:35.701218558Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-05-07T16:23:36.142235783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:23:36.145908998Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:23:37.342282807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:23:37.345981264Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:23:38.915697821Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=debug ts=2026-05-07T16:23:38.915722558Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-07T16:23:39.985965086Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:23:39.98599012Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:23:39.985998537Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:23:39.986048551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:42.129569637Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=debug ts=2026-05-07T16:23:42.709485015Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:23:43.473724379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:23:43.48033453Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:23:44.305551805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:23:44.309093201Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=warn ts=2026-05-07T16:23:44.985712213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:45.344213358Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-07T16:23:48.558709797Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-07T16:23:49.168502072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:23:49.172880689Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:23:49.985476748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:23:50.141902256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:23:50.145155196Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:23:50.431245097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:23:50.434355306Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-07T16:23:50.470972184Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:23:51.774468227Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-07T16:23:51.774490189Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=4
level=info ts=2026-05-07T16:23:51.81292809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:23:51.815997385Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:23:51.816017914Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:51.816040935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:23:51.818478612Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:23:52.704019562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:23:52.70711309Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:23:54.025001912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:23:54.059181199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:23:54.062109958Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:23:54.986291965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:23:54.987771384Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-07T16:23:54.987794527Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=info ts=2026-05-07T16:23:55.192961243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:23:55.200540733Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:23:55.200561824Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:23:55.200603313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:23:55.206904429Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:23:56.186909923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:23:56.190324493Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:23:58.181283527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:23:58.18423498Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=debug ts=2026-05-07T16:23:58.201470755Z caller=checkpoint.go:372 msg="writing series" size="312 kB"
level=info ts=2026-05-07T16:23:58.661647713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:23:58.664598266Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:23:59.798787024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:23:59.804529752Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:23:59.986097112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:00.387062245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:24:00.392567487Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:24:01.415326777Z caller=checkpoint.go:372 msg="writing series" size="957 B"
level=debug ts=2026-05-07T16:24:04.629517186Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-07T16:24:04.985670124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:05.117065595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:24:05.121682457Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:24:05.121707256Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:05.12173514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:24:05.125394318Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-07T16:24:05.423295807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49690"
level=info ts=2026-05-07T16:24:05.905799768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:24:05.909413346Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:24:07.791443922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:24:07.795985499Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-07T16:24:07.844374153Z caller=checkpoint.go:372 msg="writing series" size="645 kB"
level=info ts=2026-05-07T16:24:09.966378477Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:24:09.96641429Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:24:09.985828026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:24:09.98585307Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:24:09.98586109Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:24:09.985906917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:10.982932178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:24:10.990423178Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:24:11.057832616Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
level=debug ts=2026-05-07T16:24:11.057854577Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-07T16:24:14.272087999Z caller=checkpoint.go:372 msg="writing series" size="568 kB"
level=warn ts=2026-05-07T16:24:14.98619627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:15.668460737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:24:15.673792831Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:24:15.673821128Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:15.673846284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:24:15.676633706Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:24:17.306821918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:24:17.310107065Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:24:17.486693567Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T16:24:17.649370709Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:24:17.649454164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="728 kB" avg_comp="728 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" idle=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:24:17.666386006Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=info ts=2026-05-07T16:24:18.078515521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:24:18.081618149Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:24:18.081653912Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:18.081691046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:24:18.084668253Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:24:19.939380173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:24:19.946533653Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:24:19.986001048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:24:20.473336197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=debug ts=2026-05-07T16:24:20.705987051Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-05-07T16:24:20.70601218Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.7 MB" series=3
level=debug ts=2026-05-07T16:24:23.915686331Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-07T16:24:24.396285448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:24:24.400224914Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:24:24.986419756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:25.846650474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:24:25.851041637Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:24:27.130768279Z caller=checkpoint.go:372 msg="writing series" size="826 kB"
level=info ts=2026-05-07T16:24:27.734908736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:24:27.740427726Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:24:29.985652281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:30.194925048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:24:30.19890447Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:24:30.343956593Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T16:24:30.343979719Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-07T16:24:30.916702041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:24:30.919986033Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:24:31.404627692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:24:31.408249703Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:24:32.34518078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="628 B" avg_comp="628 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:24:32.34824997Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:24:32.348283446Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:32.348304282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:24:32.350730528Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:24:32.695029793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:24:32.6994776Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=debug ts=2026-05-07T16:24:33.558048865Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-07T16:24:33.869586097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:24:33.871166081Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:24:34.414963838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:24:34.417810235Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=warn ts=2026-05-07T16:24:34.986357112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:35.543115774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:24:35.547363804Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:24:36.772191525Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:24:36.984050288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:24:36.987303542Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:24:39.98550315Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:24:39.985530826Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:24:39.98553957Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:24:39.985558358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:24:39.986618862Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T16:24:39.989352748Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-07T16:24:39.990627506Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.004523062s
level=info ts=2026-05-07T16:24:40.155144504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:24:40.158427291Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:24:40.944548238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:24:40.950587892Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:24:42.099786716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:24:42.102574808Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:24:42.511696121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:24:42.514849892Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:24:43.299225194Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:24:43.299320791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" idle=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:24:43.312983269Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=warn ts=2026-05-07T16:24:44.98652029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:46.537310221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:24:46.541297149Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:24:47.082944854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:24:47.090665376Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:24:49.985977579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:50.038072539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:24:50.041395081Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:24:50.041414041Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:50.04143547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:24:50.044755334Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
ts=2026-05-07T16:24:50.476040691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:24:52.231086447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:24:52.234748204Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:24:54.371932633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:24:54.376213561Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:24:54.376238375Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:24:54.376288153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:24:54.379688349Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-07T16:24:54.986486525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:24:55.302683102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:24:55.307103996Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=info ts=2026-05-07T16:24:55.885316849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:24:55.891143138Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:24:58.199313991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:24:58.20605623Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-07T16:24:59.351190448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:57618"
level=warn ts=2026-05-07T16:24:59.986480478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:03.137382238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:25:03.140682943Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:25:04.657554834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:25:04.660963273Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:25:04.735137712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:25:04.739520724Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=warn ts=2026-05-07T16:25:04.986315416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:05.065612169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:25:05.068571553Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:25:06.378298315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:25:06.38164226Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:25:08.041115627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="549 B" avg_comp="549 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:25:08.044017079Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:25:09.966230444Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:25:09.966670381Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:25:09.985588776Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-07T16:25:09.985599942Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-07T16:25:09.985612479Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:25:09.985620991Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:25:09.985633249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:09.985871104Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-07T16:25:09.985880909Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:25:09.985901142Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.564µs success=true
level=debug ts=2026-05-07T16:25:09.986188041Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-07T16:25:11.03851258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:25:13.206373126Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T16:25:13.206397427Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=info ts=2026-05-07T16:25:13.953377464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:25:13.957391285Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-07T16:25:14.986059418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:16.074530067Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:25:16.374412783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:25:16.381069836Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:25:16.415170744Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-07T16:25:19.639998398Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-07T16:25:19.640026776Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=2
level=info ts=2026-05-07T16:25:19.660071978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:25:19.665213772Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:25:19.985676051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:25:20.47837935Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T16:25:21.352216221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:25:21.355359625Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:25:21.86195764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:25:21.867363525Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:25:22.844413912Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-07T16:25:24.053485909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:25:24.056554422Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:25:24.091604223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:25:24.09424175Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:25:24.307782917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:25:24.310870678Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-07T16:25:24.986380381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:25.578881508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:25:25.583079969Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=debug ts=2026-05-07T16:25:26.058071496Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-07T16:25:27.04994648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:60268"
level=info ts=2026-05-07T16:25:28.696666624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:25:28.69960066Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:25:28.77601355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:25:28.782968457Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:25:29.272631454Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
ts=2026-05-07T16:25:29.352720057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50908"
level=info ts=2026-05-07T16:25:29.645026725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:25:29.648741664Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:25:29.98629063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:30.414242535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:25:30.417486605Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:25:32.486847952Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-07T16:25:32.63022228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:25:32.633413824Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:25:33.948991733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:25:33.952395456Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:25:34.986266367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:35.520085532Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:25:35.701342851Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-07T16:25:37.726142799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:25:37.730858709Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-07T16:25:38.914981652Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-07T16:25:39.985475837Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:25:39.985501888Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:25:39.985509512Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:25:39.985509343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:41.352161568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:25:41.35881649Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:25:42.129867842Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-07T16:25:42.748583268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:25:42.752560576Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-07T16:25:44.80375942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:25:44.807717111Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:25:44.985854089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:25:45.249707426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:25:45.254016256Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=debug ts=2026-05-07T16:25:45.345802535Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:25:45.345825132Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=8
level=debug ts=2026-05-07T16:25:48.557919655Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=warn ts=2026-05-07T16:25:49.986274609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:25:50.479819566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=info ts=2026-05-07T16:25:50.738010187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:25:50.747604908Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:25:51.77268827Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:25:54.877824085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:25:54.881467844Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:25:54.986308958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:25:54.986362587Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-07T16:25:55.372947974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:25:55.376369038Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:25:55.617978019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:25:55.621387132Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:25:58.201766642Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=warn ts=2026-05-07T16:25:59.985862906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:00.616947844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:26:00.62556936Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:26:01.414909611Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-07T16:26:01.948290359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:26:01.951538992Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:26:02.320771137Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:26:04.629754816Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-07T16:26:04.986391713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:06.440312484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:26:06.44390499Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:26:06.817852781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:26:06.82273212Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=debug ts=2026-05-07T16:26:07.906245164Z caller=checkpoint.go:372 msg="writing series" size="61 MB"
level=debug ts=2026-05-07T16:26:07.906293781Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=7
level=info ts=2026-05-07T16:26:09.96620994Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:26:09.966203945Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:26:09.985438661Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:26:09.985464357Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:26:09.985474294Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:26:09.985504774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:10.03882366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:26:10.98547128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:26:10.988298121Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-07T16:26:11.059835683Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:26:11.059854403Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-07T16:26:12.060609014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:26:12.063493689Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:26:13.278611387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:26:13.284662549Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:26:14.147935387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:26:14.150962952Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:26:14.273089873Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=warn ts=2026-05-07T16:26:14.986213274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:16.032162592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:26:16.035833248Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:26:16.827016974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:26:16.834821578Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:26:17.486654165Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:26:18.761689201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:26:18.764661165Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:26:18.771849163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:26:18.77700342Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-07T16:26:19.986309879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:20.444057189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:26:20.446866795Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:26:20.446889296Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:26:20.446916935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:26:20.449120454Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-07T16:26:20.481606776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:26:20.701132079Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=info ts=2026-05-07T16:26:22.819303737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="545 B" avg_comp="545 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:26:22.823146921Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:26:23.452810141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:26:23.459542523Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:26:23.915763052Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-07T16:26:24.986183684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:26.237709213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:26:26.241301193Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-07T16:26:27.052915275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55950"
level=debug ts=2026-05-07T16:26:27.129932152Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-07T16:26:28.150955776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:26:28.156145581Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=info ts=2026-05-07T16:26:29.097732265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:26:29.100913599Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:26:29.492144386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:26:29.49721124Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-07T16:26:29.986140087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:30.344314149Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=debug ts=2026-05-07T16:26:30.616880944Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:26:33.558072999Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T16:26:33.558096747Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-07T16:26:34.985903369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:36.772176264Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-07T16:26:36.904471215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:26:36.909065003Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:26:39.986278353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:26:39.986302792Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:26:39.986311771Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:26:39.98631004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:39.986330823Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-07T16:26:42.714235052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:26:42.719200673Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:26:42.902824337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:26:42.906576738Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:26:43.200456813Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-07T16:26:43.313714232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:26:43.317422655Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:26:44.98634819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:46.414972434Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-07T16:26:48.95936984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:26:48.963216421Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:26:49.002459152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:26:49.005731625Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:26:49.629395621Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-07T16:26:49.827991033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:26:49.832438516Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-07T16:26:49.985807907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:26:50.294559556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:26:50.29973603Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
ts=2026-05-07T16:26:50.483483357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T16:26:52.392162134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="125 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:26:52.399296179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:26:52.848694916Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-07T16:26:52.848716701Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=6
level=warn ts=2026-05-07T16:26:54.985870418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:26:56.057683296Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-07T16:26:59.272211987Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=warn ts=2026-05-07T16:26:59.985832686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:00.055033956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:27:00.062651353Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:27:00.430275291Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:27:02.486690991Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=debug ts=2026-05-07T16:27:03.247383947Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:27:04.33069379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:27:04.333619445Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:27:04.986244219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:05.372076053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:27:05.375183345Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
ts=2026-05-07T16:27:05.440070488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57638"
level=debug ts=2026-05-07T16:27:05.701001049Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=info ts=2026-05-07T16:27:08.252053252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:27:08.260638618Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:27:08.277686253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:27:08.281812638Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=info ts=2026-05-07T16:27:08.409180218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:27:08.412077301Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-07T16:27:08.915524569Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:27:09.160308387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:27:09.166210882Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:27:09.166232386Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:27:09.16627725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:27:09.171693221Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:27:09.514548668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:27:09.517703441Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:27:09.966285392Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:27:09.966766711Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:27:09.986306716Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:27:09.986331318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:27:09.986342045Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:27:09.986372207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:10.066817014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:27:10.070701295Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:27:12.129881747Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=info ts=2026-05-07T16:27:13.324304026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:27:13.328101539Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-07T16:27:14.18976472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:27:14.193167699Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:27:14.266410705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:27:14.269933357Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=warn ts=2026-05-07T16:27:14.986017426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:15.262158513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:27:15.269161541Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:27:15.344582651Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=debug ts=2026-05-07T16:27:15.344605314Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-07T16:27:16.742100971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:27:16.745734388Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:27:18.55811265Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-05-07T16:27:19.986051585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:20.151314108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:27:20.158133226Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:27:20.485321621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:27:21.772239501Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=info ts=2026-05-07T16:27:21.878322264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:27:21.882448182Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-07T16:27:24.986448216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:27:24.986469865Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-07T16:27:25.49698787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:27:25.501031278Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
ts=2026-05-07T16:27:27.056893694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35370"
level=info ts=2026-05-07T16:27:27.828372066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:27:27.832054346Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:27:28.201246806Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-07T16:27:29.986057606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:27:31.415675659Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-07T16:27:34.62972393Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-07T16:27:34.986119027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:27:35.384383799Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:27:37.823276264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:27:37.828330429Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:27:37.843631066Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=info ts=2026-05-07T16:27:39.250227238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="149 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:27:39.25859889Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:27:39.986436158Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:27:39.986463422Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:27:39.986472287Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:27:39.986460957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:27:41.058543915Z caller=checkpoint.go:372 msg="writing series" size="765 kB"
level=debug ts=2026-05-07T16:27:41.058568247Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=8
level=info ts=2026-05-07T16:27:44.223913334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:27:44.228904081Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:27:44.272161718Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-07T16:27:44.57674706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:27:44.579769576Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:27:44.909616054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:27:44.915721409Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:27:44.985694544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:44.987825649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:27:44.992112584Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=info ts=2026-05-07T16:27:45.438983157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:27:45.441683628Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:27:46.438637753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:27:46.443109056Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:27:47.486994521Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-07T16:27:49.273815921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:27:49.279363151Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=debug ts=2026-05-07T16:27:49.313044567Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-07T16:27:49.98641282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:27:50.486800797Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=info ts=2026-05-07T16:27:50.649743372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:27:50.652938107Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:27:50.652959943Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:27:50.652980507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:27:50.655404687Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:27:50.700607971Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
level=debug ts=2026-05-07T16:27:53.914919942Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=warn ts=2026-05-07T16:27:54.98587735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:27:55.386157878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:27:55.391641015Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:27:55.391662287Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:27:55.39168818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:27:55.396298848Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:27:55.770749258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:27:55.77414403Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-07T16:27:57.05829971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51296"
level=debug ts=2026-05-07T16:27:57.130212663Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=info ts=2026-05-07T16:27:57.346828707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:27:57.353163175Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:27:57.353184588Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:27:57.35320858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:27:57.359173175Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:27:59.295965268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:27:59.299231362Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-07T16:27:59.986148639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:00.344072149Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T16:28:00.980458944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:28:00.983862768Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:28:01.629493359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:28:01.636373295Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:28:03.54851715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:28:03.551916241Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:28:03.558014653Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-07T16:28:04.220099525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:28:04.223488414Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:28:04.985919099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:06.772154041Z caller=checkpoint.go:372 msg="writing series" size="625 kB"
level=debug ts=2026-05-07T16:28:06.772180184Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=8
level=debug ts=2026-05-07T16:28:07.441622395Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:28:07.57708632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:28:07.580784432Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=info ts=2026-05-07T16:28:08.246800508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:28:08.258202663Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:28:08.465809429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:28:08.469149639Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:28:09.028590421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:28:09.034786038Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:28:09.966248761Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:28:09.966560176Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:28:09.985493781Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:28:09.985517012Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-07T16:28:09.985511837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:28:09.985524609Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:28:09.986650383Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-05-07T16:28:10.178772064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:28:10.184311952Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:28:10.793602793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:28:10.796617953Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=debug ts=2026-05-07T16:28:13.201330617Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-07T16:28:14.985955743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:28:15.487309243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:28:15.491612736Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-07T16:28:16.415329879Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:28:19.378177992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:28:19.3823085Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:28:19.630552749Z caller=checkpoint.go:372 msg="writing series" size="767 kB"
level=warn ts=2026-05-07T16:28:19.985863114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:28:20.488071144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:28:22.844389517Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-05-07T16:28:22.844409737Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-07T16:28:24.985889799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:28:25.253502001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:28:25.257605496Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:28:26.058708975Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
ts=2026-05-07T16:28:27.059992439Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34510"
level=info ts=2026-05-07T16:28:28.727400251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:28:28.732121825Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=debug ts=2026-05-07T16:28:29.272566042Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-07T16:28:29.272587714Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=info ts=2026-05-07T16:28:29.947836254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:28:29.952533164Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=warn ts=2026-05-07T16:28:29.985519814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:32.486571711Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-07T16:28:33.667937959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:28:33.671921454Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:28:34.293808108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:28:34.299989205Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-07T16:28:34.986011079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:35.701700959Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-05-07T16:28:37.481891922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:28:37.485485776Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:28:38.915347428Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=debug ts=2026-05-07T16:28:38.915372759Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-07T16:28:39.148591195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="149 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:28:39.156107511Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:28:39.156133072Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:28:39.15616377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="149 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:28:39.162029099Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:28:39.842079309Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:28:39.842153157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=79de45044a76a6b4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="189 kB" avg_uncomp="189 kB" idle=1 labels="{stream=\"build-e2e-snng\"}"
level=error ts=2026-05-07T16:28:39.845924981Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/79de45044a76a6b4/19e02f8d67b%3A19e030de8ae%3Aaab9cd18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-snng\"}"
level=info ts=2026-05-07T16:28:39.985941073Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:28:39.985969151Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:28:39.985978067Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:28:39.985990623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:28:40.061301089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:28:40.064298688Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:28:41.186638756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:28:41.191626067Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:28:41.362675918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:28:41.365622031Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:28:42.129597376Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-07T16:28:42.59607151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:28:42.601593188Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:28:42.948390266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:28:42.953661397Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-07T16:28:44.986369424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:45.343896207Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-07T16:28:46.571693213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:28:46.574367452Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:28:47.934786384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-07T16:28:47.936452721Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-07T16:28:47.987905971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:28:47.99571823Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-07T16:28:48.558420589Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-07T16:28:49.985499341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:28:50.490791834Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:28:51.770941614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="149 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:28:51.777367888Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-07T16:28:51.77738839Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=4
level=error ts=2026-05-07T16:28:51.779555759Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:28:53.497299756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=79de45044a76a6b4 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-snng\"}"
level=error ts=2026-05-07T16:28:53.500827932Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/79de45044a76a6b4/19e02f8d67b%3A19e030de8ae%3Aaab9cd18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-snng\"}"
level=info ts=2026-05-07T16:28:53.671322007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:28:53.675150024Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:28:54.512486051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:28:54.515915625Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=warn ts=2026-05-07T16:28:54.985836398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:28:54.988109761Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-07T16:28:54.988128897Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=1
level=info ts=2026-05-07T16:28:55.615123358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:28:55.618282007Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:28:55.618310494Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:28:55.618336647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:28:55.620802406Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:28:56.117304283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:28:56.120345407Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:28:56.120372687Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:28:56.120395518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:28:56.122678607Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:28:56.575227277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:28:56.578086934Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-07T16:28:58.201558783Z caller=checkpoint.go:372 msg="writing series" size="370 kB"
level=info ts=2026-05-07T16:28:58.949628639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:28:58.952631056Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-07T16:28:58.952652234Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:28:58.952676458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:28:58.955104146Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:28:59.986126403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:01.414950352Z caller=checkpoint.go:372 msg="writing series" size="957 B"
level=debug ts=2026-05-07T16:29:04.629671294Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-07T16:29:04.986211198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:05.765027173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="628 B" avg_comp="628 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:29:05.768367536Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:29:07.414769302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:29:07.418781389Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-07T16:29:07.843675165Z caller=checkpoint.go:372 msg="writing series" size="645 kB"
level=info ts=2026-05-07T16:29:09.96626243Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:29:09.966290418Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:29:09.986382809Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:29:09.986404118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:29:09.986412104Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:29:09.986427206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:11.057774214Z caller=checkpoint.go:372 msg="writing series" size="486 kB"
level=debug ts=2026-05-07T16:29:11.057798223Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-07T16:29:11.15072949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:29:11.154447576Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:29:11.180749845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:29:11.183748613Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:29:11.308007603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:29:11.311718823Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:29:14.067423902Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-07T16:29:14.2723507Z caller=checkpoint.go:372 msg="writing series" size="682 kB"
level=warn ts=2026-05-07T16:29:14.985959602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:15.500467332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:29:15.503691369Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:29:15.870672003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="149 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:29:15.878383885Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-07T16:29:17.486901932Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:29:19.59207825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=79de45044a76a6b4 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-snng\"}"
level=error ts=2026-05-07T16:29:19.595707971Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/79de45044a76a6b4/19e02f8d67b%3A19e030de8ae%3Aaab9cd18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-snng\"}"
level=warn ts=2026-05-07T16:29:19.98612836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:29:20.493351916Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-99347fe9 10.129.0.83:7946"
level=debug ts=2026-05-07T16:29:20.70333475Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-05-07T16:29:20.703357831Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.8 MB" series=3
level=info ts=2026-05-07T16:29:20.89524627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:29:20.899564921Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=info ts=2026-05-07T16:29:22.222402885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:29:22.225949736Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:29:22.225972151Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:29:22.225994927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:29:22.228527569Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:29:23.864665082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:29:23.867809372Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:29:23.87885815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:29:23.881686893Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=debug ts=2026-05-07T16:29:23.914735165Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-07T16:29:24.985872899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:26.351957884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:29:26.357343981Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=debug ts=2026-05-07T16:29:27.130140272Z caller=checkpoint.go:372 msg="writing series" size="833 kB"
level=debug ts=2026-05-07T16:29:27.727037709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:29:27.847178252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:29:27.851034019Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:29:28.190232782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:29:28.19312893Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:29:28.193151994Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:29:28.193174013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:29:28.195349253Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-07T16:29:29.985488413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:30.344857365Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-05-07T16:29:30.344882258Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-07T16:29:30.72806321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:29:30.736026447Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:29:31.457120468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:29:31.462842863Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-07T16:29:32.689288512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:29:32.696130381Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:29:33.557775783Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-07T16:29:34.137465048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:29:34.141423281Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:29:34.141448448Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:29:34.141472584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:29:34.14452831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-07T16:29:34.985782185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:35.338453284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:29:35.341817438Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-07T16:29:35.454188543Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:36590"
level=debug ts=2026-05-07T16:29:36.7727698Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:29:37.121590415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:29:37.125137959Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:29:37.857472642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:29:37.863223352Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:29:38.766676383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:29:38.770036445Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:29:39.323630775Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:29:39.985906064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:29:39.985940184Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:29:39.985950226Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:29:39.985973148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:29:39.986038574Z caller=checkpoint.go:384 msg="flushing series" totalSize="191 kB" series=2
level=info ts=2026-05-07T16:29:39.997455189Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-07T16:29:39.998761598Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012729592s
level=info ts=2026-05-07T16:29:41.617205721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:29:41.622244819Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=warn ts=2026-05-07T16:29:44.986076328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:45.331240283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:29:45.334726141Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:29:45.693149669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:29:45.696355276Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:29:46.034396313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="549 B" avg_comp="549 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:29:46.039191525Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:29:47.738202719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:29:47.741384214Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:29:48.569925747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:29:48.573182164Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:29:49.985619618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:29:50.301848685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:29:50.305685225Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-07T16:29:50.494801739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=warn ts=2026-05-07T16:29:54.985917656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:29:57.064220048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55420"
level=info ts=2026-05-07T16:29:57.910224692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:29:57.914524346Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-07T16:29:59.36699128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:37974"
level=warn ts=2026-05-07T16:29:59.986139404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:00.064449261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:30:00.068045215Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:30:00.92649883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:00.930447818Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:00.930471296Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:00.93049376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:00.933313838Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:30:03.086004185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=79de45044a76a6b4 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-snng\"}"
level=error ts=2026-05-07T16:30:03.090005928Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/79de45044a76a6b4/19e02f8d67b%3A19e030de8ae%3Aaab9cd18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-snng\"}"
level=info ts=2026-05-07T16:30:04.619310051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:30:04.622355711Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-07T16:30:04.986247112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:30:05.457226334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43710"
level=info ts=2026-05-07T16:30:06.019471756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:30:06.022477876Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:30:08.133523538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:30:08.137122259Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=info ts=2026-05-07T16:30:09.966241801Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:30:09.966774042Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=debug ts=2026-05-07T16:30:09.968074625Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-07T16:30:09.968130789Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975745 n=1
level=debug ts=2026-05-07T16:30:09.968188206Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975745 n=1
level=warn ts=2026-05-07T16:30:09.985498502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:09.985496262Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-07T16:30:09.985523427Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:30:09.98554138Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:30:09.985549524Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-07T16:30:09.985626076Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-07T16:30:09.985636241Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-07T16:30:09.98565399Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.965µs success=true
level=debug ts=2026-05-07T16:30:09.985907367Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-07T16:30:10.462688524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:30:10.706703101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="149 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:30:10.713790078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:30:11.522981456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:30:11.526906303Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:30:12.280718796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:12.285364077Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-07T16:30:13.079657199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:30:13.084914995Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:30:13.206807421Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-07T16:30:13.206832477Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.4 MB" series=2
level=warn ts=2026-05-07T16:30:14.986283324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:30:16.415023425Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-07T16:30:16.600612484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:30:16.604382022Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-07T16:30:17.449710488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:30:17.457545477Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:30:17.457567787Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:17.45760658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:30:17.463748028Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:30:18.486640583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:30:18.490629508Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:30:18.49065005Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:18.490675251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:30:18.494076023Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=debug ts=2026-05-07T16:30:19.647023958Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-07T16:30:19.647050975Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=2
level=warn ts=2026-05-07T16:30:19.986519527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:20.494174102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
ts=2026-05-07T16:30:20.496518791Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=error ts=2026-05-07T16:30:20.497394282Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:30:20.497415174Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:20.497435958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:30:20.499957274Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:30:21.871372163Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-07T16:30:22.183386331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:30:22.186875498Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=debug ts=2026-05-07T16:30:22.843772166Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-07T16:30:22.971137669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:30:22.974425656Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:30:24.986165699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:25.572007881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:30:25.576388884Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=debug ts=2026-05-07T16:30:26.057681709Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-07T16:30:28.016659488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:30:28.020839655Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:30:28.247490146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:30:28.255038207Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-07T16:30:29.272193204Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=warn ts=2026-05-07T16:30:29.985592263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:30.922081789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:30:30.925402843Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-07T16:30:31.025766888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:30:31.028917275Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-07T16:30:31.967262735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:30:31.970873672Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-07T16:30:32.486356442Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-07T16:30:34.885240534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:30:34.888869889Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:30:34.888892515Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:34.888915329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:30:34.891508882Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-07T16:30:34.985645532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:35.141081976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="17 MB" avg_uncomp="8.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:30:35.147643003Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-07T16:30:35.46064614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:48066"
level=debug ts=2026-05-07T16:30:35.700321131Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-07T16:30:36.258060017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:30:36.267275436Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-07T16:30:36.414750568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:30:36.419309339Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=debug ts=2026-05-07T16:30:38.915197194Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=debug ts=2026-05-07T16:30:39.181229155Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:30:39.986278649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:30:39.98631115Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:30:39.986321605Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:30:39.986320285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:30:40.152841626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:30:40.156234876Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-07T16:30:42.12941414Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-07T16:30:43.634553399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:30:43.637339453Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:30:43.895107818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:30:43.898730512Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-07T16:30:44.985727451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:30:45.345069992Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-07T16:30:45.345093386Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=8
level=info ts=2026-05-07T16:30:46.581465678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:30:46.584495829Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:30:46.584517075Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:46.58453949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:30:46.586711755Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:30:48.558153859Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-07T16:30:49.986315494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:30:50.499487007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:30:51.429692229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:30:51.434433559Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-07T16:30:51.743224001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:30:51.746804592Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:30:51.772137905Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-07T16:30:52.237384809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:30:52.240626508Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:30:53.187493366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=79de45044a76a6b4 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-snng\"}"
level=error ts=2026-05-07T16:30:53.191006795Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/79de45044a76a6b4/19e02f8d67b%3A19e030de8ae%3Aaab9cd18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-snng\"}"
level=info ts=2026-05-07T16:30:54.889977796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="149 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:30:54.897929667Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-07T16:30:54.986164529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:30:54.986232418Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-05-07T16:30:57.0677935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:37694"
level=info ts=2026-05-07T16:30:57.337033792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:30:57.340705169Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:30:57.340728699Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:30:57.340751241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:30:57.343370881Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:30:57.416756696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:30:57.420041496Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-07T16:30:58.200822304Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=info ts=2026-05-07T16:30:59.7488547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:30:59.756069523Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:30:59.985786309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:01.315102091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:31:01.319440654Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:31:01.319461243Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:31:01.319483271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:31:01.323204008Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:31:01.414662911Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-07T16:31:01.447788067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:31:01.451094474Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=info ts=2026-05-07T16:31:04.561922583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:31:04.567721325Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:31:04.629085233Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-07T16:31:04.901305298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:31:04.90432295Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-07T16:31:04.986455793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:31:05.610871644Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:31:06.007390639Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:31:06.844548652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:31:06.847756622Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:31:06.847777175Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:31:06.847798135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:31:06.85043258Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=debug ts=2026-05-07T16:31:07.906787018Z caller=checkpoint.go:372 msg="writing series" size="61 MB"
level=debug ts=2026-05-07T16:31:07.906818596Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=7
level=info ts=2026-05-07T16:31:09.966218076Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:31:09.96672606Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:31:09.98588092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:31:09.985901404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:31:09.985910065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:31:09.985957253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:11.051377821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:31:11.054821361Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:31:11.059154715Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-07T16:31:11.059167051Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=info ts=2026-05-07T16:31:11.95329349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:31:11.957512412Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=info ts=2026-05-07T16:31:12.071048579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:31:12.07529342Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-07T16:31:12.82147163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:31:12.824955487Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:31:13.693270268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="563 B" avg_comp="563 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:31:13.696054972Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-07T16:31:14.141375575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:31:14.145767681Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:31:14.145791038Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:31:14.145814913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:31:14.149376982Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=debug ts=2026-05-07T16:31:14.272523438Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=warn ts=2026-05-07T16:31:14.985888889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:15.53279891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:31:15.539428598Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:31:16.231925694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:31:16.237335381Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-07T16:31:17.486879412Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-07T16:31:17.869842153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-07T16:31:17.872881809Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e02ef874e%3A19e02eff7cd%3A8ce2ec5b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-07T16:31:19.986367526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:31:20.501585939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-4b6tr-80b98376 10.129.0.81:7946"
level=debug ts=2026-05-07T16:31:20.700938486Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-05-07T16:31:21.125940944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:31:21.129591306Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:31:23.329907465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:31:23.333958554Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=debug ts=2026-05-07T16:31:23.914653855Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-07T16:31:24.985747308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:25.034904994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:31:25.037835473Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-07T16:31:25.414507763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="17 MB" avg_uncomp="8.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:31:25.420744101Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-07T16:31:25.651970809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:31:25.65619977Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=debug ts=2026-05-07T16:31:27.129691213Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-07T16:31:28.272482256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-07T16:31:28.275805628Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e02efdf02%3A19e02f09534%3Ae3b832b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-07T16:31:29.986089116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:31:30.344193316Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=debug ts=2026-05-07T16:31:33.557389172Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-07T16:31:33.557409532Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-07T16:31:34.532123253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-07T16:31:34.535844758Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e02ef89be%3A19e02eff7cd%3A10468ed3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-07T16:31:34.98650243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:31:36.771814488Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-07T16:31:36.848093347Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-07T16:31:39.274119651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:31:39.27851664Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:31:39.278540588Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-07T16:31:39.278567675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:31:39.28201186Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-07T16:31:39.442497124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=79de45044a76a6b4 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-snng\"}"
level=error ts=2026-05-07T16:31:39.446464718Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/79de45044a76a6b4/19e02f8d67b%3A19e030de8ae%3Aaab9cd18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-snng\"}"
level=warn ts=2026-05-07T16:31:39.985964727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:39.985969952Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:31:39.985991292Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:31:39.985999549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-07T16:31:39.986041366Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-07T16:31:42.770912789Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-07T16:31:43.201165884Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-07T16:31:43.337495446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-07T16:31:43.340804939Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e02ef879d%3A19e02eff7cd%3A980ee88a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-07T16:31:43.463325509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="486 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-07T16:31:43.471248332Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e02ef813c%3A19e02f8579f%3A4a9ec3f8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-07T16:31:44.98621501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:46.063182815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="758 kB" avg_comp="758 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-07T16:31:46.067644276Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e02ef807a%3A19e032d28ad%3Ad6405fea?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-07T16:31:46.41464717Z caller=checkpoint.go:372 msg="writing series" size="657 B"
level=info ts=2026-05-07T16:31:48.293638875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="126 kB" avg_uncomp="126 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-07T16:31:48.297151186Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e02ef8863%3A19e02f15ffe%3A38b5fda3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-07T16:31:49.629747874Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-07T16:31:49.985557342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-07T16:31:50.503449932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-792xr-1e3267b5 10.128.0.66:7946"
level=info ts=2026-05-07T16:31:51.302890598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d3c3953344b36e49 immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-pyos\"}"
level=error ts=2026-05-07T16:31:51.306429459Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d3c3953344b36e49/19e02f7962b%3A19e02f7b779%3A8dd69dcd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-pyos\"}"
level=info ts=2026-05-07T16:31:51.843478313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7ca308d06025fde5 immediate=false num_chunks=1 total_comp="3.3 kB" avg_comp="3.3 kB" total_uncomp="6.9 kB" avg_uncomp="6.9 kB" full=1 labels="{stream=\"build-e2e-kyhh\"}"
level=error ts=2026-05-07T16:31:51.846683824Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7ca308d06025fde5/19e02f7d6b8%3A19e02f9dc34%3A61a221dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kyhh\"}"
level=info ts=2026-05-07T16:31:52.549551933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="149 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-07T16:31:52.55716684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e02ef87bc%3A19e02efebda%3Aba28a9e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-07T16:31:52.692527792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-07T16:31:52.695939309Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e02f82e83%3A19e02f999cc%3A6073672b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-07T16:31:52.847147192Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-07T16:31:52.847178868Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=6
level=info ts=2026-05-07T16:31:54.028897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="545 B" avg_comp="545 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-07T16:31:54.033363541Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e02eff7cc%3A19e02f0914f%3A8c67ec1d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-07T16:31:54.985662818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:31:55.561279665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fafcbac47e64f2f3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-xfwk-managed\"}"
level=error ts=2026-05-07T16:31:55.564439649Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fafcbac47e64f2f3/19e03001c33%3A19e03011bac%3Abe265592?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-xfwk-managed\"}"
level=info ts=2026-05-07T16:31:55.639650235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="2.0 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-07T16:31:55.646786402Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e02ef81b3%3A19e03016547%3A9dcd0797?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-07T16:31:55.702006994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="567 B" avg_comp="567 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-07T16:31:55.704994885Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e02efdf02%3A19e02efdf03%3A940a72ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-07T16:31:56.058248633Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
ts=2026-05-07T16:31:57.07178591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49872"
level=info ts=2026-05-07T16:31:57.398024572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c70f3592536c42a7 immediate=false num_chunks=1 total_comp="766 kB" avg_comp="766 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-qxdw\"}"
level=error ts=2026-05-07T16:31:57.402539745Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c70f3592536c42a7/19e02f918e3%3A19e030a62ac%3A34d6b4b2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qxdw\"}"
level=info ts=2026-05-07T16:31:58.555012131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-07T16:31:58.55888292Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e02ef87a5%3A19e0339cef6%3A3e6e67b6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-07T16:31:59.272569506Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=warn ts=2026-05-07T16:31:59.986171403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:00.972226262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-07T16:32:00.977878789Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e02ef86f1%3A19e0319c350%3A93bba02f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-07T16:32:02.485878818Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=debug ts=2026-05-07T16:32:03.451375795Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-07T16:32:04.985980337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-07T16:32:05.700606185Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=info ts=2026-05-07T16:32:07.184616295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.5 MB" avg_uncomp="4.5 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-07T16:32:07.18942289Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e02efdea5%3A19e02fe2401%3A7cc44f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-07T16:32:07.731896842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9895b571c520f663 immediate=false num_chunks=1 total_comp="765 kB" avg_comp="765 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" full=1 labels="{stream=\"build-e2e-rded\"}"
level=error ts=2026-05-07T16:32:07.736328337Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9895b571c520f663/19e02f991fb%3A19e030993a4%3A8164ed8d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rded\"}"
level=info ts=2026-05-07T16:32:07.836645846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.6 MB" avg_comp="1.3 MB" total_uncomp="17 MB" avg_uncomp="8.4 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-07T16:32:07.843571336Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e02efdf02%3A19e030d6276%3Acb3f2dfc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-07T16:32:08.914612343Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-07T16:32:09.002099469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-07T16:32:09.005199206Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e02eff7cc%3A19e02f0d35d%3Ade19c24f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-07T16:32:09.966194143Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-07T16:32:09.967639923Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.85:11211 count=1
level=info ts=2026-05-07T16:32:09.986158715Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-07T16:32:09.986181375Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-07T16:32:09.986189855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-07T16:32:09.986193559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-07T16:32:10.514330965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-07T16:32:10.518374835Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e02efdf02%3A19e02f1067c%3A7c7013ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"