level=info ts=2026-05-09T18:39:03.988651398Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-09T18:39:03.988733879Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-09T18:39:03.989593679Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-09T18:39:03.990670995Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-09T18:39:03.991087039Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T18:39:03.991179484Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.73 inf=eth0
level=info ts=2026-05-09T18:39:03.991745367Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-2e237f23
ts=2026-05-09T18:39:03.991824052Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-09T18:39:03.991838703Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.73 advertisePort=7946
level=debug ts=2026-05-09T18:39:03.991862703Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.73 advertisePort=7946
level=info ts=2026-05-09T18:39:03.992017482Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-09T18:39:03.99618873Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T18:39:03.996211188Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-09T18:39:03.997412796Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-09T18:39:03.997473641Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-05-09T18:39:03.997806685Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T18:39:03.997828669Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T18:39:03.997844092Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.828817ms
level=error ts=2026-05-09T18:39:03.997864404Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-09T18:39:03.997874934Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-05-09T18:39:03.998658817Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-09T18:39:03.998769468Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-09T18:39:03.998765481Z
level=debug ts=2026-05-09T18:39:03.99888022Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-09T18:39:03.99891123Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-09T18:39:03.998925317Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-09T18:39:03.998930876Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-09T18:39:03.998930092Z
level=debug ts=2026-05-09T18:39:03.99893719Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-05-09T18:39:03.999932624Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-09T18:39:04.000001599Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.73 inf=eth0
ts=2026-05-09T18:39:04.000884383Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T18:39:04.000902433Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-09T18:39:04.013846103Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-05-09T18:39:04.013892438Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-09T18:39:04.013908333Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-09T18:39:04.013927659Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-09T18:39:04.01392167Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-05-09T18:39:04.013932119Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-09T18:39:04.013942226Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-09T18:39:04.014016029Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-09T18:39:04.01402625Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-09T18:39:04.014025483Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-05-09T18:39:04.014027652Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-09T18:39:04.014035317Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-09T18:39:04.014172773Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-09T18:39:04.014175768Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-05-09T18:39:04.01418109Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-05-09T18:39:04.014193354Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-05-09T18:39:04.014203784Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-09T18:39:04.014231523Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-09T18:39:04.01423589Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-05-09T18:39:04.014238926Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-05-09T18:39:04.014242003Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-09T18:39:04.014363028Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-09T18:39:04.014372934Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-09T18:39:04.014378483Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=info ts=2026-05-09T18:39:04.014383576Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-09T18:39:04.014406482Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-09T18:39:04.014417763Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T18:39:04.014454028Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=21.203µs success=true
level=info ts=2026-05-09T18:39:04.01446704Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-09T18:39:04.014774809Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=369.672µs errors=false
level=info ts=2026-05-09T18:39:04.014787175Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-09T18:39:04.01490946Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=503.741µs errors=false
level=info ts=2026-05-09T18:39:04.014924182Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-09T18:39:04.014931572Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=526.11µs
level=info ts=2026-05-09T18:39:04.014998673Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-09T18:39:04.015023227Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-09T18:39:04.015038882Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=19.723940474s
level=info ts=2026-05-09T18:39:04.015120808Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=debug ts=2026-05-09T18:39:04.015126883Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-09T18:39:04.01513271Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-05-09T18:39:04.015131008Z caller=loki.go:615 msg="Loki started" startup_time=84.100746ms
level=debug ts=2026-05-09T18:39:04.015286492Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-05-09T18:39:04.015508286Z caller=wal.go:173 msg=started component=wal
ts=2026-05-09T18:39:05.767745067Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T18:39:05.767768873Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:39:08.974361262Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T18:39:08.974381867Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T18:39:09.016080874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:39:13.422398958Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T18:39:13.422421629Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T18:39:14.015430024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:39:19.016132951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:39:20.049782991Z caller=logging.go:137 msg="GET /ready (503) 86.329µs"
ts=2026-05-09T18:39:23.606007384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.89:7946"
ts=2026-05-09T18:39:23.608769961Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.92:7946: dial tcp 10.130.0.92:7946: connect: connection refused"
ts=2026-05-09T18:39:23.609953548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.91:7946"
ts=2026-05-09T18:39:23.612207526Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.74:7946"
ts=2026-05-09T18:39:23.61395485Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.73:7946: dial tcp 10.129.0.73:7946: connect: connection refused"
ts=2026-05-09T18:39:23.614585774Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.74:7946: dial tcp 10.129.0.74:7946: connect: connection refused"
level=info ts=2026-05-09T18:39:23.614594909Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=19.616713932s
level=warn ts=2026-05-09T18:39:24.015161468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:39:29.015947209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:39:30.048833772Z caller=logging.go:137 msg="GET /ready (503) 70.356µs"
level=info ts=2026-05-09T18:39:34.015215686Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-09T18:39:34.015251728Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-09T18:39:34.015261197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:39:34.015273916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:39:39.015462162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:39:44.015307313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:39:49.015559353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:39:54.015324479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:39:55.446708557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T18:39:59.015901714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:39:59.069045082Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T18:39:59.069085592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:39:59.074804925Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:40:00.414011106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T18:40:03.997547825Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:40:03.997562013Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:40:04.016071831Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T18:40:04.016075567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:40:04.016114722Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:40:04.016122296Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:40:09.015878848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:40:11.18722513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:40:11.196611001Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-09T18:40:14.009892757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49136"
level=warn ts=2026-05-09T18:40:14.015397383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:40:19.0162096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:40:24.015661897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:40:24.965756834Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-09T18:40:25.449109212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=warn ts=2026-05-09T18:40:29.015481978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:40:29.710686192Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T18:40:34.015963412Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T18:40:34.015974777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:40:34.015992845Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:40:34.016004195Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T18:40:34.909303373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:40:34.915668719Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:40:39.015559904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:40:43.49285981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T18:40:44.016151873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:40:49.015886385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:40:54.015240539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:40:55.452717784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=warn ts=2026-05-09T18:40:59.015433152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:40:59.107833975Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T18:41:03.9975594Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:41:03.998354908Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:41:04.015698021Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:41:04.015712595Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:41:04.015720708Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:41:04.015710845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:41:05.678924973Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T18:41:09.016206371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:41:14.015610697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:41:19.015930404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:41:24.015895393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:41:25.455022424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T18:41:26.712481661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:41:26.720154817Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:41:29.015888695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:41:31.254723279Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:41:33.976758356Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T18:41:33.978494092Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T18:41:34.01562945Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:41:34.015655817Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:41:34.01566434Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:41:34.015712973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:41:39.015627801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:41:44.015613285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:41:49.015405412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:41:50.755112726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60392"
level=warn ts=2026-05-09T18:41:54.015585768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:41:55.45659239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T18:41:59.015884722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:42:03.997577098Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:42:03.997999656Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:42:04.015275781Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T18:42:04.01528445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:42:04.015294963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:42:04.015304591Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:42:04.087627766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T18:42:04.087688511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="158 kB" avg_comp="158 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:42:04.090916171Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T18:42:08.606214944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:42:08.612549471Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:42:09.01521783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:42:14.015742311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:42:14.562702803Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T18:42:19.015597677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:42:20.744347392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:42:20.749725101Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:42:24.015660973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:42:25.458128316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T18:42:29.015489944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:42:34.015804128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:42:34.015818832Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:42:34.01583935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:42:34.015846569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:42:34.665742948Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:42:39.015477669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:42:44.015629099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:42:44.024040841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:56932"
level=info ts=2026-05-09T18:42:44.179524569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:42:44.183657334Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:42:49.015672783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:42:54.016028364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:42:55.459538232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T18:42:59.015488302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:43:03.897937503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:43:03.903927529Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T18:43:03.997525254Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:43:03.997935818Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:43:04.015187687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:43:04.015206191Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:43:04.015214115Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:43:04.015221157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:43:08.787562394Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:43:09.016142705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:43:13.998287556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-09T18:43:14.015543498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:43:19.015346594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:43:20.762676926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60178"
level=warn ts=2026-05-09T18:43:24.015341902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:43:25.460373749Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T18:43:29.015787148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:43:34.015954709Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T18:43:34.01594789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:43:34.015975045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:43:34.015982942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:43:37.696191519Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T18:43:38.613643132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:43:38.619891632Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:43:39.01579912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:43:44.01566123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:43:44.028836906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:34920"
level=info ts=2026-05-09T18:43:44.254083424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:43:44.26051452Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:43:49.015893777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:43:54.015980845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:43:55.461542953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T18:43:59.015902038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:44:03.997507176Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:44:03.99786079Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:44:04.015123688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:44:04.015140858Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:44:04.015148687Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:44:04.015156579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:44:04.016224171Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T18:44:04.016345398Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-09T18:44:04.016352033Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T18:44:04.016366141Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.667µs success=true
level=debug ts=2026-05-09T18:44:04.016533757Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
ts=2026-05-09T18:44:06.955998002Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:44834"
level=debug ts=2026-05-09T18:44:08.372069539Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T18:44:09.015768017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:44:11.217789177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-09T18:44:11.889400811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:44:12.726285759Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-09T18:44:14.015590835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:44:17.08199222Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T18:44:19.016139306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:44:20.767535425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35336"
level=debug ts=2026-05-09T18:44:21.436974783Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-05-09T18:44:24.01533601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:44:25.463311467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T18:44:25.790905294Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T18:44:26.096935106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:44:26.102744234Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:44:29.015374993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:44:30.146356459Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-09T18:44:34.015867067Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:44:34.015894055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:44:34.015901901Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:44:34.015897783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:44:34.304430521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="23 MB" avg_uncomp="23 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:44:34.310917606Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:44:34.501304173Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-09T18:44:36.959245789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:41668"
level=debug ts=2026-05-09T18:44:38.855796518Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-09T18:44:39.016221241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:44:43.211258782Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-05-09T18:44:44.015855081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:44:46.509841226Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:44:47.565668317Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T18:44:49.015327701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:44:50.768442313Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37922"
level=debug ts=2026-05-09T18:44:51.920158634Z caller=checkpoint.go:372 msg="writing series" size="434 kB"
level=debug ts=2026-05-09T18:44:51.920180328Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-05-09T18:44:54.015898471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:44:55.465333824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T18:44:56.275158629Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-09T18:44:59.015826362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:45:00.63030602Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=debug ts=2026-05-09T18:45:01.702528703Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-09T18:45:03.997582265Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:45:03.997611192Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-09T18:45:03.999929113Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T18:45:03.999974636Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975946 n=1
level=debug ts=2026-05-09T18:45:04.000014732Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975946 n=1
level=info ts=2026-05-09T18:45:04.015985392Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T18:45:04.015990345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:45:04.016006634Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:45:04.016015705Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:45:04.984957484Z caller=checkpoint.go:372 msg="writing series" size="328 kB"
ts=2026-05-09T18:45:06.962213793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:42810"
level=warn ts=2026-05-09T18:45:09.016138843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:45:09.339400163Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T18:45:13.694455758Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T18:45:14.015273785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:45:16.279716426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:45:16.288005415Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T18:45:18.049812685Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-09T18:45:18.298756722Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:45:19.015169593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:45:22.403864984Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=warn ts=2026-05-09T18:45:24.015311885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:45:25.468490939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:45:26.758578053Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-09T18:45:28.685128049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:45:28.694166773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:45:29.015619554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:45:31.114085698Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-09T18:45:34.015878743Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:45:34.015906716Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:45:34.015914942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:45:34.015924234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:45:35.469042833Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-09T18:45:36.965303536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:43166"
level=warn ts=2026-05-09T18:45:39.015969307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:45:39.824071877Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-09T18:45:39.824107803Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-05-09T18:45:44.015890958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:45:44.178298955Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-09T18:45:48.533420104Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T18:45:49.015999003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:45:50.772410471Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44800"
level=debug ts=2026-05-09T18:45:52.88757292Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-09T18:45:53.712879651Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-09T18:45:53.714561549Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T18:45:54.015569222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:45:54.033764183Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T18:45:55.471298147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T18:45:57.242999195Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-09T18:45:59.01614078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:46:01.597925911Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=info ts=2026-05-09T18:46:03.997735555Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:46:03.998647414Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:46:04.015998129Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:46:04.016014407Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T18:46:04.016007175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:46:04.016022943Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:46:05.952570013Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-09T18:46:06.405136709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:46:06.408882161Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T18:46:06.968710755Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:52808"
level=warn ts=2026-05-09T18:46:09.015648142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:46:10.30739436Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=info ts=2026-05-09T18:46:11.014215053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:46:11.020787684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:46:11.020807775Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T18:46:11.020829712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:46:11.026829329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:46:14.015931909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:46:14.662743822Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-09T18:46:19.015452163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:46:19.016600235Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=debug ts=2026-05-09T18:46:23.37162523Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-09T18:46:24.015795712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:46:25.473981906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:46:27.726797526Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-09T18:46:29.015926815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:46:29.419936322Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T18:46:29.596562554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:46:29.607066992Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:46:32.081936086Z caller=checkpoint.go:372 msg="writing series" size="335 kB"
level=debug ts=2026-05-09T18:46:32.081959478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-05-09T18:46:34.01541034Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T18:46:34.015407249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:46:34.015433202Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:46:34.01544418Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:46:35.217104401Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T18:46:36.436266598Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-09T18:46:39.015900912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:46:40.791644891Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-05-09T18:46:44.016006979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:46:44.044231673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39356"
level=debug ts=2026-05-09T18:46:45.146506211Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-09T18:46:49.015946254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:46:49.50044932Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-09T18:46:53.855759102Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T18:46:54.016054701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:46:55.477290646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T18:46:58.212036628Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T18:46:58.212084553Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-09T18:46:59.016170363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:47:00.710923121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:47:00.719278712Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:47:02.015979152Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T18:47:02.402123033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:47:02.405945633Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T18:47:02.56526775Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-09T18:47:03.997302276Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:47:03.998341652Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-09T18:47:04.015727817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:47:04.01572514Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:47:04.015748185Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:47:04.015756878Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:47:06.920512705Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-09T18:47:09.015348857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:11.274776212Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-09T18:47:14.015682384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:15.629815899Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-09T18:47:19.015742463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:19.985653754Z caller=checkpoint.go:372 msg="writing series" size="526 kB"
level=debug ts=2026-05-09T18:47:20.021855137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T18:47:24.015186375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:24.339460076Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-09T18:47:25.479397608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T18:47:28.694934874Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-09T18:47:29.015806923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:32.099598373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:47:33.049356349Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=debug ts=2026-05-09T18:47:33.049376396Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-09T18:47:34.015483545Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:47:34.015508707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:47:34.01551577Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:47:34.015534469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:37.404258779Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T18:47:39.01614532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:41.758803487Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-09T18:47:44.016127603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:47:45.892705437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:47:45.901315478Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T18:47:46.113984526Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-09T18:47:49.015568172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:50.472286063Z caller=checkpoint.go:372 msg="writing series" size="5.6 MB"
level=debug ts=2026-05-09T18:47:50.472309817Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.7 MB" series=4
ts=2026-05-09T18:47:50.783499202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59422"
level=info ts=2026-05-09T18:47:51.597806557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:47:51.604156195Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:47:54.015463644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:54.823312107Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
ts=2026-05-09T18:47:55.481981709Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=warn ts=2026-05-09T18:47:59.015537845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:47:59.177897273Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-09T18:47:59.588449606Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:48:00.865146357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T18:48:03.53316375Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-09T18:48:03.997541821Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:48:03.997927663Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:48:04.01525503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:48:04.01527291Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:48:04.015291243Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:48:04.015288038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:48:07.888269872Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=warn ts=2026-05-09T18:48:09.015915458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:48:12.242797874Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-09T18:48:14.015595275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:48:16.597925197Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-09T18:48:19.016152427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:48:20.953072382Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T18:48:24.01581312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:48:25.307255708Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
ts=2026-05-09T18:48:25.484873576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:48:27.455310587Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:48:29.015527581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:48:29.661933963Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=info ts=2026-05-09T18:48:34.01601823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:48:34.016041857Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T18:48:34.016031278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:48:34.01605015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:48:34.017115944Z caller=checkpoint.go:384 msg="flushing series" totalSize="701 kB" series=9
level=info ts=2026-05-09T18:48:34.022445024Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-09T18:48:34.025814729Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009326671s
level=info ts=2026-05-09T18:48:37.313357624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:48:37.324817493Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:48:39.016135396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:48:43.97995631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:48:43.983816176Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:48:43.983835133Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T18:48:43.983856986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:48:43.986610896Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:48:44.015743943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:48:44.053981639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:38232"
level=warn ts=2026-05-09T18:48:49.015609773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:48:53.813252352Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:48:54.015450084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:48:55.488046403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:48:57.719431118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T18:48:59.016059887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:49:01.179607543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:49:01.183028392Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T18:49:03.997726574Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:49:03.997731605Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:49:04.015133052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:49:04.015158645Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:49:04.015168855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:49:04.015177509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:49:04.016232936Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T18:49:04.016367043Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-09T18:49:04.016376282Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T18:49:04.016395751Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.977µs success=true
level=debug ts=2026-05-09T18:49:04.016623549Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
ts=2026-05-09T18:49:06.986345102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:46776"
level=debug ts=2026-05-09T18:49:08.372017167Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T18:49:09.016130403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:49:12.727149828Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-09T18:49:14.015712192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:49:17.08168189Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T18:49:19.015955516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:49:20.492911853Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:49:21.436831968Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-09T18:49:24.015242027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:49:25.491607995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T18:49:25.791576287Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-09T18:49:29.016036919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:49:29.089311205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:49:29.099817013Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:49:30.146359447Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-09T18:49:34.015124805Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:49:34.015149634Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:49:34.015157382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:49:34.015161365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:49:34.501275651Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-09T18:49:37.493898273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T18:49:37.4957469Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T18:49:37.890872241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:49:37.894179503Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T18:49:38.855558658Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-09T18:49:39.015925739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:49:43.210953068Z caller=checkpoint.go:372 msg="writing series" size="329 kB"
level=warn ts=2026-05-09T18:49:44.015964856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:49:47.565994267Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T18:49:49.015941522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:49:50.794327789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48664"
level=debug ts=2026-05-09T18:49:51.920612262Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=debug ts=2026-05-09T18:49:51.920636282Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=debug ts=2026-05-09T18:49:52.800279917Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:49:54.015201308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:49:55.495044445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:49:56.275524905Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-09T18:49:59.01553373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:50:00.63000192Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=info ts=2026-05-09T18:50:03.998120669Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:50:03.999089632Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:50:04.015452793Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:50:04.015470266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:50:04.015479146Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:50:04.015487575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:50:04.984632477Z caller=checkpoint.go:372 msg="writing series" size="348 kB"
level=warn ts=2026-05-09T18:50:09.016178848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:50:09.339424994Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T18:50:10.512306619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:50:10.522666719Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:50:13.694714972Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T18:50:14.015824928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:50:14.062389999Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:57452"
level=debug ts=2026-05-09T18:50:16.179251444Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T18:50:18.049147023Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-09T18:50:18.176532045Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:50:19.015619609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:50:22.2926695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:50:22.300444315Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T18:50:22.403979884Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=warn ts=2026-05-09T18:50:24.015680104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:50:25.498123652Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:50:26.759129692Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-09T18:50:29.015633494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:50:31.11396405Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=debug ts=2026-05-09T18:50:31.113984187Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=warn ts=2026-05-09T18:50:34.015976115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:50:34.015978693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:50:34.016000038Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:50:34.01600701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:50:35.469147147Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
ts=2026-05-09T18:50:36.992845878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47082"
level=warn ts=2026-05-09T18:50:39.015213028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:50:39.824192414Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-09T18:50:44.016017533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:50:44.178420361Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-09T18:50:45.764401424Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:50:48.533740182Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T18:50:49.01574674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:50:50.799459191Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39994"
level=debug ts=2026-05-09T18:50:52.88835598Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-09T18:50:54.015603251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:50:55.501416613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T18:50:57.242753004Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-09T18:50:59.0156016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:50:59.149879855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:50:59.159043609Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:51:01.598176641Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=debug ts=2026-05-09T18:51:03.913078224Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T18:51:03.997717523Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:51:03.998080636Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:51:04.015489652Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:51:04.01550449Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:51:04.01551162Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:51:04.015526861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:51:05.952860157Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-05-09T18:51:06.995565807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:51436"
level=warn ts=2026-05-09T18:51:09.015204101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:51:10.307272926Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=warn ts=2026-05-09T18:51:14.015678461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:51:14.06668242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:55898"
level=debug ts=2026-05-09T18:51:14.661925578Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-09T18:51:15.518699306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:51:15.523229189Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:51:19.015845407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:51:19.016974697Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-09T18:51:21.694965868Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:51:23.37152669Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-09T18:51:24.015545222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:51:25.504752522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:51:27.726789846Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-09T18:51:29.0159951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:51:32.082135984Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-09T18:51:32.082158386Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=info ts=2026-05-09T18:51:34.015177074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:51:34.015204093Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:51:34.015214232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:51:34.01520934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:51:36.436346723Z caller=checkpoint.go:372 msg="writing series" size="4.7 kB"
ts=2026-05-09T18:51:36.998825116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:51700"
level=warn ts=2026-05-09T18:51:39.015486283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:51:40.791647139Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-09T18:51:41.118180499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T18:51:41.777880312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:51:41.786289841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:51:44.015599726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:51:45.145994201Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-05-09T18:51:48.818648227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:51:49.01598204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:51:49.500991026Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
ts=2026-05-09T18:51:50.804186379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52068"
level=debug ts=2026-05-09T18:51:53.85579498Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T18:51:54.01520681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:51:55.507595305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T18:51:58.211362889Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T18:51:58.211384246Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-09T18:51:59.015256616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:02.565626761Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-09T18:52:03.997638666Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:52:03.998295581Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:52:04.015763787Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:52:04.015780583Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T18:52:04.01577663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:52:04.015788169Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T18:52:04.306387155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:52:04.309653465Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T18:52:06.920725756Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-09T18:52:09.015229247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:11.274768034Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-09T18:52:14.016024042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:52:14.073479834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:60862"
level=debug ts=2026-05-09T18:52:15.630540665Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-05-09T18:52:17.105424186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T18:52:19.015416987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:19.985547025Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-05-09T18:52:22.033818472Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:52:24.015573091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:24.339786359Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-09T18:52:25.511142947Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:52:28.694996161Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-09T18:52:29.016265775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:33.049347566Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=debug ts=2026-05-09T18:52:33.049369304Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-09T18:52:33.453446795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:52:33.462168795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:52:33.46219317Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T18:52:33.462215277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:52:33.468205306Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T18:52:34.015254087Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:52:34.015278977Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:52:34.015294083Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:52:34.015354023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:37.404212153Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-09T18:52:39.016207378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:41.759334493Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-09T18:52:44.0155121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:46.113429118Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-09T18:52:48.229228388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:52:48.241175103Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T18:52:48.286200428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:52:48.289226873Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:52:49.015738933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:50.473431098Z caller=checkpoint.go:372 msg="writing series" size="6.6 MB"
level=debug ts=2026-05-09T18:52:50.473452072Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.8 MB" series=4
level=warn ts=2026-05-09T18:52:54.016196387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:54.17057256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:52:54.824159048Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
ts=2026-05-09T18:52:55.512911305Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T18:52:59.0155014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:52:59.178883275Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-09T18:53:03.533181646Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-09T18:53:03.998547983Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:53:03.999579687Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:53:04.015965675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:53:04.015983721Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:53:04.01599155Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:53:04.016014226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:53:07.888509471Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=warn ts=2026-05-09T18:53:09.015221399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:53:12.243476804Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-09T18:53:14.015523417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:53:15.285920032Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T18:53:15.287482409Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T18:53:15.930455099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:53:15.938157937Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:53:16.598143396Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-09T18:53:19.015879525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:53:20.952428138Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T18:53:24.015303215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:53:25.307381799Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
ts=2026-05-09T18:53:25.515209524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T18:53:25.783386185Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:53:29.016142599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:53:29.662299088Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-09T18:53:31.602655188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:53:31.606722096Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T18:53:34.015571401Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:53:34.015597226Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:53:34.015605338Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:53:34.01560172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:53:34.016680889Z caller=checkpoint.go:384 msg="flushing series" totalSize="759 kB" series=9
level=info ts=2026-05-09T18:53:34.022252363Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-09T18:53:34.023635571Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007069399s
ts=2026-05-09T18:53:37.011383575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:35346"
level=warn ts=2026-05-09T18:53:39.015628482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:53:44.01601087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:53:49.016126521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:53:52.095512966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T18:53:54.015767219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:53:55.517606383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T18:53:59.015971982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:54:01.216883413Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T18:54:03.998550805Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:54:03.999494274Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:54:04.01591473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:54:04.01593432Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:54:04.015926573Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T18:54:04.015944118Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:54:04.01595844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:54:04.016053545Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-09T18:54:04.01606171Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T18:54:04.01608174Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.456µs success=true
level=debug ts=2026-05-09T18:54:04.016273258Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-09T18:54:08.371595678Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T18:54:09.016142061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:54:12.723554878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:54:12.726685935Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=error ts=2026-05-09T18:54:12.73688377Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:54:14.015792933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:54:14.08373985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49472"
level=debug ts=2026-05-09T18:54:17.083182055Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T18:54:19.015801141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:54:21.435957645Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-09T18:54:22.908807854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:54:22.913523661Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:54:24.015653531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:54:25.519455455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T18:54:25.791573633Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-05-09T18:54:28.261501048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:54:29.015632412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:54:30.14635234Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T18:54:34.015567166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:54:34.015573955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:54:34.015592731Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:54:34.015600468Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:54:34.030829268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T18:54:34.501230117Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-09T18:54:37.01762043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:46248"
level=debug ts=2026-05-09T18:54:38.855060365Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-09T18:54:39.01563783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:54:43.210652891Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=warn ts=2026-05-09T18:54:44.015178072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:54:44.086716524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:37444"
level=debug ts=2026-05-09T18:54:47.565358485Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-09T18:54:49.015767687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:54:51.920634829Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=debug ts=2026-05-09T18:54:51.920655437Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=12
level=warn ts=2026-05-09T18:54:54.015983714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:54:55.021659497Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T18:54:55.522652953Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:54:56.275225197Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-09T18:54:59.015456679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:54:59.462146422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:54:59.495423568Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:55:00.629459189Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=info ts=2026-05-09T18:55:03.997664746Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:55:03.998128506Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:55:04.015711821Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:55:04.015742498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:55:04.015749679Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:55:04.015739415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:55:04.985431067Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=info ts=2026-05-09T18:55:05.208205079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:55:05.211846121Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:55:05.211866226Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T18:55:05.211886374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:55:05.214533975Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T18:55:07.020381561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:39934"
level=warn ts=2026-05-09T18:55:09.015989327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:55:09.339170898Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T18:55:12.926187573Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T18:55:13.694435446Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=debug ts=2026-05-09T18:55:13.694455255Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-09T18:55:14.016034575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:55:18.048915117Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-09T18:55:19.015983131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:55:21.249507068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:55:21.253137742Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T18:55:21.805699618Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:55:22.404478334Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=warn ts=2026-05-09T18:55:24.015664061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:55:25.524440643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T18:55:26.758585033Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-05-09T18:55:29.015836872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:55:31.113994271Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-05-09T18:55:34.015128279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:55:34.015153025Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:55:34.015161403Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:55:34.015153656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:55:35.468697645Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-09T18:55:39.015614415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:55:39.823293338Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=warn ts=2026-05-09T18:55:44.015667406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:55:44.177942026Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-09T18:55:44.36990238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:55:44.379677312Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:55:48.532918135Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-09T18:55:49.015944933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:55:50.829534204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49796"
level=debug ts=2026-05-09T18:55:51.092391116Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T18:55:52.71087597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:55:52.714145237Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T18:55:52.88727391Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-09T18:55:54.015646636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:55:55.526333815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:55:56.164654238Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:55:57.242974208Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-09T18:55:59.015557493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:56:01.597181054Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=debug ts=2026-05-09T18:56:01.59720445Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-09T18:56:03.998352753Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:56:03.999364576Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:56:04.015890831Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:56:04.01590984Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:56:04.015917515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:56:04.015910278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:56:05.952304247Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T18:56:09.015908449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:56:10.307263572Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=warn ts=2026-05-09T18:56:14.016037397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:56:14.661705346Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-09T18:56:19.015625159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:56:19.01669479Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
ts=2026-05-09T18:56:20.830791542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48684"
level=debug ts=2026-05-09T18:56:21.545682914Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T18:56:21.545738611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T18:56:21.557984896Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T18:56:23.371851994Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-09T18:56:24.016140033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:56:25.528467819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T18:56:27.726488492Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=warn ts=2026-05-09T18:56:29.01625301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:56:29.905832204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T18:56:29.907775308Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T18:56:32.081974968Z caller=checkpoint.go:372 msg="writing series" size="366 kB"
level=info ts=2026-05-09T18:56:34.015802848Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:56:34.015829954Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:56:34.015838466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:56:34.015868359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:56:36.393595909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T18:56:36.402181679Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T18:56:36.436257472Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
ts=2026-05-09T18:56:37.030817172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:37134"
level=info ts=2026-05-09T18:56:37.409504808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:56:37.416078298Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:56:39.015656406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:56:40.790717862Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-09T18:56:40.79074302Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=warn ts=2026-05-09T18:56:44.016003944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:56:45.145807053Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-09T18:56:45.608352141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:56:45.612379954Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T18:56:45.614467275Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:56:49.015972423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:56:49.501080794Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-09T18:56:53.855276912Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T18:56:54.015916634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:56:55.530673802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T18:56:58.211386276Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-09T18:56:58.211406839Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-09T18:56:59.015590984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:57:00.5389368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T18:57:00.553800732Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T18:57:02.564902098Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-09T18:57:03.997600213Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:57:03.997960842Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:57:04.01563888Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:57:04.015664608Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:57:04.015673288Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:57:04.015670069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:06.920479834Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
ts=2026-05-09T18:57:07.034613593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:33140"
level=warn ts=2026-05-09T18:57:09.015919713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:11.274873116Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=warn ts=2026-05-09T18:57:14.016203567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:57:14.099803443Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:38312"
level=debug ts=2026-05-09T18:57:15.62991509Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-05-09T18:57:19.01559131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:19.985136726Z caller=checkpoint.go:372 msg="writing series" size="564 kB"
ts=2026-05-09T18:57:20.835559431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41738"
level=debug ts=2026-05-09T18:57:21.222145161Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:57:21.506567099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T18:57:24.015817353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:24.339516216Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-09T18:57:24.33954117Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-09T18:57:25.534164076Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T18:57:25.994007184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:57:26.002116186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:57:28.69423474Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T18:57:29.015802039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:33.049045451Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=info ts=2026-05-09T18:57:34.015800802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:57:34.015829983Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:57:34.015837974Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:57:34.015828807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:37.404053213Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T18:57:38.653471968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:57:38.661184692Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:57:39.016153243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:41.758650178Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=warn ts=2026-05-09T18:57:44.015951331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:57:44.102948547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:47220"
level=debug ts=2026-05-09T18:57:46.113251149Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-09T18:57:47.209376676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T18:57:47.217222934Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T18:57:49.01600408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:50.255780403Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:57:50.473263059Z caller=checkpoint.go:372 msg="writing series" size="7.1 MB"
level=debug ts=2026-05-09T18:57:50.473285803Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.6 MB" series=6
ts=2026-05-09T18:57:50.837348368Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57558"
level=warn ts=2026-05-09T18:57:54.016025999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:54.823417007Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
ts=2026-05-09T18:57:55.536108174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T18:57:59.016158971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:57:59.178537888Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=debug ts=2026-05-09T18:58:03.533430544Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=debug ts=2026-05-09T18:58:03.974811899Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T18:58:03.997209117Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:58:03.998260074Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T18:58:04.015946682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:58:04.015967903Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T18:58:04.01596583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:58:04.015975841Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:58:07.88835286Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=warn ts=2026-05-09T18:58:09.015448204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:58:09.677647672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:58:09.685254256Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T18:58:12.242238223Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-05-09T18:58:14.015186781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:58:16.597537806Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-09T18:58:17.807890158Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T18:58:19.016031202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:58:20.952435586Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-05-09T18:58:24.015810922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:58:25.306922045Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
ts=2026-05-09T18:58:25.538258525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=warn ts=2026-05-09T18:58:29.0161551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:58:29.662288016Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-09T18:58:34.016081616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T18:58:34.016119336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:58:34.016135351Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:58:34.016146328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T18:58:34.0171286Z caller=checkpoint.go:384 msg="flushing series" totalSize="957 kB" series=9
level=info ts=2026-05-09T18:58:34.022809653Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-09T18:58:34.024334148Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00811087s
level=info ts=2026-05-09T18:58:35.383221521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:58:35.38953061Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T18:58:39.01597948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:58:42.712145438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T18:58:42.720997735Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T18:58:44.015723217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T18:58:49.0162328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:58:49.394245703Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:58:50.785191922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T18:58:54.01584081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:58:55.540966809Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=warn ts=2026-05-09T18:58:59.015426587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:59:03.123163928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:59:03.133370708Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:59:03.13340492Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T18:59:03.13343791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:59:03.140611692Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T18:59:03.997540696Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T18:59:03.997863157Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T18:59:04.015459792Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:59:04.015478233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:59:04.015486708Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:59:04.015517512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:59:04.015539194Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T18:59:04.015654166Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-09T18:59:04.015659963Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T18:59:04.015677563Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.47µs success=true
level=debug ts=2026-05-09T18:59:04.015860265Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-09T18:59:07.522685033Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T18:59:09.015728668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:59:11.028918285Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-05-09T18:59:13.976483977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:59:13.98743145Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:59:14.01563402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:59:14.112124578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:48086"
level=debug ts=2026-05-09T18:59:14.535855278Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-09T18:59:17.459046629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:59:18.041883576Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T18:59:19.015838508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T18:59:20.717406492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T18:59:20.721899294Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T18:59:20.845385705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41580"
level=debug ts=2026-05-09T18:59:21.549078499Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=warn ts=2026-05-09T18:59:24.015823403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:59:25.055786361Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
ts=2026-05-09T18:59:25.543973512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T18:59:28.109031208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T18:59:28.117316148Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T18:59:28.561480196Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-05-09T18:59:29.015259894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:59:32.06858915Z caller=checkpoint.go:372 msg="writing series" size="5.0 kB"
level=debug ts=2026-05-09T18:59:33.912204179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T18:59:34.015760627Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T18:59:34.015787049Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T18:59:34.015797516Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T18:59:34.015810666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:59:35.575281549Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-09T18:59:37.049288066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:58806"
level=warn ts=2026-05-09T18:59:39.016011191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:59:39.081171682Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-09T18:59:42.587950537Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-09T18:59:44.015998873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:59:46.094403781Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-09T18:59:47.942447245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T18:59:47.951968949Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T18:59:49.016150928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T18:59:49.601282259Z caller=checkpoint.go:372 msg="writing series" size="400 kB"
level=debug ts=2026-05-09T18:59:49.60130383Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-05-09T18:59:50.790572197Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T18:59:53.107595113Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-09T18:59:54.015295605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T18:59:55.547203708Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T18:59:56.613978031Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-09T18:59:59.015363185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:00:00.120300233Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
level=debug ts=2026-05-09T19:00:03.626770143Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-09T19:00:03.997751877Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:00:03.998780746Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-09T19:00:04.000012459Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T19:00:04.000053805Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975947 n=1
level=debug ts=2026-05-09T19:00:04.000113386Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975947 n=1
level=info ts=2026-05-09T19:00:04.015254939Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:00:04.015268565Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:00:04.015276075Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:00:04.015281563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:00:06.470933329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:00:06.476142821Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:00:07.133051808Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-09T19:00:09.016034186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:00:09.273395987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:00:09.275032812Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T19:00:10.647826024Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-09T19:00:10.647847705Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=6
level=info ts=2026-05-09T19:00:12.590607025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:00:12.597183844Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:00:14.01545803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:00:14.119285137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35750"
level=debug ts=2026-05-09T19:00:14.145973714Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T19:00:17.652726847Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-09T19:00:19.015473952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:00:21.159512657Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-09T19:00:23.8392133Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:00:24.015983308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:00:24.665765536Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
ts=2026-05-09T19:00:25.550901226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:00:28.172172439Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-09T19:00:29.015904574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:00:31.679268839Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T19:00:34.015893566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:00:34.015894471Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:00:34.015919395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:00:34.015927338Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:00:35.053444641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:00:35.060878829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:00:35.184906508Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=debug ts=2026-05-09T19:00:38.691436273Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=warn ts=2026-05-09T19:00:39.016161182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:00:42.199251332Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-09T19:00:44.016053761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:00:44.122802137Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:37124"
level=debug ts=2026-05-09T19:00:45.704983369Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-09T19:00:49.01590955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:00:49.21091268Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-09T19:00:50.321890056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:00:50.32943191Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:00:52.717862655Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-09T19:00:54.015673271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:00:55.554191633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:00:56.223658789Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=debug ts=2026-05-09T19:00:58.335055021Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:00:59.015627346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:00:59.374274443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:00:59.382834021Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:00:59.730630252Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-09T19:01:01.64417854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:01:03.237299456Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-09T19:01:03.997578477Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:01:03.997608959Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:01:04.015118608Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:01:04.015136261Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:01:04.01514433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:01:04.015171029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:06.743323674Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-09T19:01:09.015993826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:10.249913657Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=debug ts=2026-05-09T19:01:10.249936534Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=17
level=debug ts=2026-05-09T19:01:13.7568296Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T19:01:14.01597999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:01:17.162538081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:01:17.170518042Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:01:17.262981172Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=warn ts=2026-05-09T19:01:19.015579379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:20.770058279Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-09T19:01:24.015962766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:24.276170863Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
ts=2026-05-09T19:01:25.556783462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:01:27.78287077Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-09T19:01:29.016255758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:31.289565844Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-09T19:01:32.883310394Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:01:34.015542961Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:01:34.015553377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:34.015576813Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:01:34.015587849Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:01:34.795456797Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-05-09T19:01:37.058132784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:32796"
level=debug ts=2026-05-09T19:01:38.302053943Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=debug ts=2026-05-09T19:01:38.302082694Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-09T19:01:39.016079084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:41.808347109Z caller=checkpoint.go:372 msg="writing series" size="6.8 kB"
level=info ts=2026-05-09T19:01:41.961251655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:01:41.969976087Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:01:41.970002374Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:01:41.970033953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:01:41.973122937Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:01:42.011556971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:01:42.018477891Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:01:44.01565072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:45.315342129Z caller=checkpoint.go:372 msg="writing series" size="324 kB"
level=debug ts=2026-05-09T19:01:48.821515343Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-09T19:01:49.01584122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:51.341860415Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:01:52.328613028Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=warn ts=2026-05-09T19:01:54.01605712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:01:55.558157677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:01:55.834599807Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-09T19:01:56.013294061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:01:56.018622943Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:01:57.79059976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:01:57.797285388Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:01:58.917393064Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:01:59.015538654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:01:59.342270272Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-09T19:01:59.342296282Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=6
level=debug ts=2026-05-09T19:02:02.847970555Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-09T19:02:03.998152961Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:02:03.998765295Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:02:04.015118929Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:02:04.015142147Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:02:04.015152169Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:02:04.015186562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:02:06.353706114Z caller=checkpoint.go:372 msg="writing series" size="342 kB"
level=warn ts=2026-05-09T19:02:09.01599624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:02:09.860640874Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-09T19:02:13.367018202Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=warn ts=2026-05-09T19:02:14.016079104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:02:16.873421063Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-09T19:02:19.015924151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:02:20.380064522Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
level=debug ts=2026-05-09T19:02:20.380086951Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-05-09T19:02:20.863254252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44174"
level=info ts=2026-05-09T19:02:23.536364648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:02:23.548738444Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:02:23.548764365Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:02:23.548799068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:02:23.555777963Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:02:23.886919047Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-09T19:02:24.015301287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:02:25.56084778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:02:25.641676242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:02:25.645358796Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:02:27.392760722Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T19:02:29.01520729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:02:30.899444445Z caller=checkpoint.go:372 msg="writing series" size="405 kB"
level=info ts=2026-05-09T19:02:34.015200814Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:02:34.015235788Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:02:34.015245736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:02:34.015262776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:02:34.268449313Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:02:34.406429062Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-09T19:02:37.912334684Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-09T19:02:39.015692678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:02:39.728816915Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:02:41.418963052Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-09T19:02:42.181968849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:02:42.188080749Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:02:43.150573725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:02:43.157205475Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:02:44.01550002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:02:44.130990804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:48254"
level=debug ts=2026-05-09T19:02:44.92569494Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-09T19:02:48.432398856Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T19:02:49.015924208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:02:50.864356853Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53238"
level=debug ts=2026-05-09T19:02:51.942963893Z caller=checkpoint.go:372 msg="writing series" size="7.7 MB"
level=debug ts=2026-05-09T19:02:51.942986399Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.5 MB" series=9
level=warn ts=2026-05-09T19:02:54.015539887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:02:55.444879735Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
ts=2026-05-09T19:02:55.563513029Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:02:58.951740571Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-09T19:02:59.016085841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:03:02.45768296Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=debug ts=2026-05-09T19:03:03.927918587Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:03:03.997860327Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:03:03.99836209Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:03:04.015830824Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:03:04.015851379Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:03:04.015861061Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:03:04.01586392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:03:05.369004482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:03:05.376835871Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:03:05.96471228Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
ts=2026-05-09T19:03:07.065458275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:37542"
level=warn ts=2026-05-09T19:03:09.015959198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:03:09.471193637Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=debug ts=2026-05-09T19:03:12.97760138Z caller=checkpoint.go:372 msg="writing series" size="303 kB"
level=debug ts=2026-05-09T19:03:12.977625608Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-09T19:03:14.015998884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:03:14.134282304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:37754"
level=debug ts=2026-05-09T19:03:16.483733206Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-09T19:03:19.015214663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:03:19.990206283Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=debug ts=2026-05-09T19:03:22.026708883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:03:23.496533848Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T19:03:24.015175463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:03:25.167003948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:03:25.17051654Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T19:03:25.565787148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:03:27.003805748Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-09T19:03:29.015508057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:03:30.509987111Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-09T19:03:32.242949364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:03:32.249530978Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:03:33.255442142Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:03:34.015721802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:03:34.015747788Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:03:34.01575691Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:03:34.015752109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:03:34.016844666Z caller=checkpoint.go:384 msg="flushing series" totalSize="136 kB" series=5
level=info ts=2026-05-09T19:03:34.115180259Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-09T19:03:34.122155343Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.106359138s
level=warn ts=2026-05-09T19:03:39.01551405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:03:44.015562127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:03:49.015737936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:03:50.868946456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39094"
level=warn ts=2026-05-09T19:03:54.015774281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:03:55.568104171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=warn ts=2026-05-09T19:03:59.015780668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:04:01.712524363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:04:01.727272267Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:04:03.997550638Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:04:03.998283026Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:04:04.01574985Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:04:04.015767857Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:04:04.015775398Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:04:04.015778257Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T19:04:04.015811577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:04:04.015908351Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-09T19:04:04.015922317Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:04:04.015940891Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.724µs success=true
level=debug ts=2026-05-09T19:04:04.016175922Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
ts=2026-05-09T19:04:07.069980523Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:46772"
level=debug ts=2026-05-09T19:04:07.523415348Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-09T19:04:09.016175521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:04:09.213272413Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:04:11.029390915Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-05-09T19:04:12.977879325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:04:12.991860653Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:04:13.208791634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:04:13.213405312Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:04:14.015749881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:04:14.536611105Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-09T19:04:18.042682803Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T19:04:19.016171289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:04:20.23029233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:04:20.232616118Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:04:20.870831331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37762"
level=debug ts=2026-05-09T19:04:21.549489403Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-09T19:04:24.015826934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:04:25.055398408Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
ts=2026-05-09T19:04:25.570472932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:04:28.561956652Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-09T19:04:29.015342208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:04:32.068381667Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-09T19:04:34.015316234Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:04:34.015308028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:04:34.015346704Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:04:34.015355254Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:04:35.574621642Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-09T19:04:36.026275221Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:04:39.01527588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:04:39.081551075Z caller=checkpoint.go:372 msg="writing series" size="385 kB"
level=debug ts=2026-05-09T19:04:42.587775047Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-09T19:04:42.587798875Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-05-09T19:04:44.015700055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:04:46.095048962Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-09T19:04:49.016111145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:04:49.601158684Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=debug ts=2026-05-09T19:04:53.108027793Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-09T19:04:53.415007141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:04:53.424628455Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:04:54.015890286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:04:55.572264362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:04:56.613549765Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-09T19:04:59.015598608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:00.121257189Z caller=checkpoint.go:372 msg="writing series" size="687 kB"
level=info ts=2026-05-09T19:05:00.594883158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:05:00.598990554Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:05:03.627524949Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=debug ts=2026-05-09T19:05:03.627557761Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-09T19:05:03.997222579Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:05:03.998283193Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:05:04.015796328Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:05:04.015816084Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:05:04.015823779Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:05:04.01584115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:05.894471178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:05:07.133948999Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=debug ts=2026-05-09T19:05:07.460820661Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:05:09.016006699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:05:10.528860023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:05:10.535483172Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:05:10.535504753Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:05:10.535528596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:05:10.542192368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:05:10.65192523Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-09T19:05:10.651950997Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=2
level=warn ts=2026-05-09T19:05:14.016011844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:14.146356525Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T19:05:17.653332396Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-09T19:05:19.015879599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:05:20.876296169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41484"
level=debug ts=2026-05-09T19:05:21.159460064Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-09T19:05:23.497115978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:05:23.510665279Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:05:24.016071255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:24.665930919Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
ts=2026-05-09T19:05:25.573179662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:05:28.172876725Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-09T19:05:29.016282271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:31.679009089Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:05:34.01562776Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:05:34.015638211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:34.015658541Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:05:34.015668677Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:05:35.185483872Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-09T19:05:38.69196726Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T19:05:39.015882083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:42.198200284Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-09T19:05:42.841173341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:05:44.01587185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:45.705365737Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-09T19:05:49.01551097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:49.211611604Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-09T19:05:52.717903263Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-09T19:05:52.806207478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:05:52.814877486Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:05:52.84195337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:05:52.848380848Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:05:54.015888703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:05:55.575222518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:05:55.939985202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:05:55.947064637Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:05:56.224163173Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-09T19:05:59.015387155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:05:59.730786133Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-09T19:06:03.237945376Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=debug ts=2026-05-09T19:06:03.997319211Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:06:03.998361653Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-09T19:06:04.015818552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:06:04.015840734Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:06:04.015859684Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:06:04.015873914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:06:04.350307099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:06:06.743766351Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-09T19:06:08.87085646Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:06:09.015364173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:06:10.250248286Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=debug ts=2026-05-09T19:06:10.250280397Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=17
level=debug ts=2026-05-09T19:06:13.756692163Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T19:06:14.015659725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:06:17.264019216Z caller=checkpoint.go:372 msg="writing series" size="424 kB"
level=warn ts=2026-05-09T19:06:19.016031861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:06:20.770379887Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-09T19:06:24.015492229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:06:24.276587078Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
ts=2026-05-09T19:06:25.577730301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:06:27.7827906Z caller=checkpoint.go:372 msg="writing series" size="287 kB"
level=warn ts=2026-05-09T19:06:29.015984051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:06:31.289159955Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=info ts=2026-05-09T19:06:34.015129838Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:06:34.01515674Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:06:34.015164258Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:06:34.015161684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:06:34.796456543Z caller=checkpoint.go:372 msg="writing series" size="659 kB"
level=debug ts=2026-05-09T19:06:34.796475785Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
ts=2026-05-09T19:06:37.085587902Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:42326"
level=debug ts=2026-05-09T19:06:38.302577212Z caller=checkpoint.go:372 msg="writing series" size="425 kB"
level=warn ts=2026-05-09T19:06:39.015479759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:06:39.135946199Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:06:41.80901124Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-09T19:06:44.015342486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:06:44.155191653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:55692"
level=debug ts=2026-05-09T19:06:45.315605732Z caller=checkpoint.go:372 msg="writing series" size="742 kB"
level=debug ts=2026-05-09T19:06:45.315630748Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-09T19:06:48.821669093Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-09T19:06:49.016009262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:06:50.885303168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53892"
level=debug ts=2026-05-09T19:06:51.548142889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:06:52.207163685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:06:52.214684561Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:06:52.327895998Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-09T19:06:52.737994291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:06:52.744884494Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:06:54.015908629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:06:54.361027323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:06:54.364506558Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T19:06:55.580142298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:06:55.835277183Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T19:06:59.015759457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:06:59.343495315Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-09T19:06:59.343518716Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=debug ts=2026-05-09T19:07:02.847771155Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=info ts=2026-05-09T19:07:03.997560636Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:07:03.997577232Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:07:04.016117882Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:07:04.016141533Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:07:04.016150244Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:07:04.016141167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:07:05.038318024Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:07:06.355186891Z caller=checkpoint.go:372 msg="writing series" size="644 kB"
level=warn ts=2026-05-09T19:07:09.015834219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:07:09.860538372Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-09T19:07:13.367784828Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=debug ts=2026-05-09T19:07:13.367815444Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-09T19:07:14.015614557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:07:14.15911568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:53594"
level=debug ts=2026-05-09T19:07:16.874480311Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-09T19:07:19.015875774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:07:20.380495297Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
ts=2026-05-09T19:07:20.886674159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:40264"
level=debug ts=2026-05-09T19:07:23.886949604Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-09T19:07:24.015950071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:07:25.583926213Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:07:26.454435261Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:07:27.39339364Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T19:07:29.015517757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:07:30.899687033Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=debug ts=2026-05-09T19:07:30.899709877Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-05-09T19:07:31.28294447Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:07:34.015697995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:07:34.015704393Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:07:34.0157249Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:07:34.015735347Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:07:34.405992084Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:07:36.103707343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:07:36.11134466Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:07:37.912787132Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T19:07:38.698999058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:07:38.707022213Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:07:39.015693548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:07:41.419300252Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=warn ts=2026-05-09T19:07:44.016015048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:07:44.162584151Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:56810"
level=debug ts=2026-05-09T19:07:44.92527014Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-09T19:07:46.430124462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:07:46.43755787Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:07:48.432545158Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T19:07:49.015511649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:07:50.888117125Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52392"
level=debug ts=2026-05-09T19:07:51.948622167Z caller=checkpoint.go:372 msg="writing series" size="8.8 MB"
level=debug ts=2026-05-09T19:07:51.948649258Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.0 MB" series=6
level=warn ts=2026-05-09T19:07:54.015778005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:07:55.444966618Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
ts=2026-05-09T19:07:55.586301498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:07:58.952256426Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-09T19:07:59.015757944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:08:02.458187844Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-09T19:08:03.997909995Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:08:03.99942445Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:08:04.015865Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:08:04.015892751Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:08:04.015904473Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:08:04.015950511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:08:05.964590331Z caller=checkpoint.go:372 msg="writing series" size="427 kB"
level=debug ts=2026-05-09T19:08:06.64473534Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:08:09.015950183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:08:09.471057221Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=debug ts=2026-05-09T19:08:12.978162406Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-05-09T19:08:12.978183979Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-09T19:08:14.015674548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:08:14.166505569Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39744"
level=debug ts=2026-05-09T19:08:16.483759771Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-09T19:08:19.016132842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:08:19.990732303Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-09T19:08:23.497180732Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T19:08:24.016102979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:08:25.414935388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:08:25.417058287Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T19:08:25.588676689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:08:27.003409406Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-09T19:08:27.364221839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:08:27.368545055Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:08:27.368566668Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:08:27.368589341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:08:27.371602266Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:08:29.015893481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:08:30.510067659Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-09T19:08:30.653488897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:08:30.66055604Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:08:31.733557355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="190 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:08:31.739940993Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:08:34.015271734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:08:34.01527974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:08:34.015299293Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:08:34.015308175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:08:34.016455843Z caller=checkpoint.go:384 msg="flushing series" totalSize="336 kB" series=5
level=info ts=2026-05-09T19:08:34.022683982Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-09T19:08:34.023639347Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007548911s
level=debug ts=2026-05-09T19:08:34.418560139Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:08:37.677912264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:08:37.681695112Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:08:39.015357358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:08:44.015820805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:08:49.01614754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:08:54.015193522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:08:55.597699339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=warn ts=2026-05-09T19:08:59.015799413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:01.667133609Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:09:03.99760312Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:09:03.99860095Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:09:04.016056865Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:09:04.016076242Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:09:04.016084044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:09:04.016068142Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T19:09:04.016126923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:09:04.016383633Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-09T19:09:04.016396656Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:09:04.016421013Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.941µs success=true
level=debug ts=2026-05-09T19:09:04.0166525Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-05-09T19:09:07.478541818Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T19:09:09.015895985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:10.940765506Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-05-09T19:09:12.750931441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:09:12.75413577Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:09:14.015928455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:14.402343429Z caller=checkpoint.go:372 msg="writing series" size="507 kB"
level=debug ts=2026-05-09T19:09:17.8638029Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T19:09:19.015412783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:21.325609125Z caller=checkpoint.go:372 msg="writing series" size="557 kB"
level=debug ts=2026-05-09T19:09:21.325638319Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T19:09:22.080492239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:09:22.087983093Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:09:22.088006225Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:09:22.088037258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:09:22.096770664Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:09:24.015622496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:24.785960242Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
ts=2026-05-09T19:09:25.601373231Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:09:26.684939445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:09:26.69146308Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:09:27.231794115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:09:28.247946677Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-09T19:09:28.422631706Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:09:29.016133662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:31.709837606Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T19:09:34.015319048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:09:34.015322598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:34.015359858Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:09:34.015368242Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:09:35.170557291Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-09T19:09:38.632347431Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-09T19:09:39.01552208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:09:39.968145972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:09:39.97736596Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:09:42.093698758Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-09T19:09:42.093728623Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-09T19:09:44.015222747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:45.555972392Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-09T19:09:49.015288055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:49.017630813Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
ts=2026-05-09T19:09:50.900286734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39852"
level=debug ts=2026-05-09T19:09:52.478670866Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-09T19:09:54.015729777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:09:55.60389904Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:09:55.940241547Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-09T19:09:57.091989142Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:09:59.015559413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:09:59.401701688Z caller=checkpoint.go:372 msg="writing series" size="941 kB"
level=debug ts=2026-05-09T19:09:59.401721902Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-09T19:10:02.554217737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="214 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:10:02.563900665Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:10:02.863019013Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=info ts=2026-05-09T19:10:03.997668275Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:10:03.998419078Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:10:04.015874709Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:10:04.015895202Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:10:04.015903214Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:10:04.015898665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:10:06.324482775Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-09T19:10:09.015495429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:10:09.816164093Z caller=checkpoint.go:372 msg="writing series" size="29 MB"
level=debug ts=2026-05-09T19:10:09.816191224Z caller=checkpoint.go:384 msg="flushing series" totalSize="29 MB" series=3
level=info ts=2026-05-09T19:10:11.276506652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:10:11.280249856Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:10:13.248246806Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T19:10:14.015913831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:10:14.190798281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:40492"
level=info ts=2026-05-09T19:10:15.410546279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:10:15.417294104Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:10:16.709805572Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
level=debug ts=2026-05-09T19:10:18.252318405Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:10:19.015654265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:10:20.171382658Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-09T19:10:20.901494178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41978"
level=debug ts=2026-05-09T19:10:23.63270839Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=debug ts=2026-05-09T19:10:23.919542281Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:10:24.01548371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:10:25.605790458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:10:27.094550982Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-09T19:10:29.015691269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:10:30.555914903Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:10:34.015699202Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:10:34.015727319Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:10:34.015734909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:10:34.015724682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:10:34.016916752Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=debug ts=2026-05-09T19:10:37.478587499Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-09T19:10:39.015986119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:10:40.940045477Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-09T19:10:44.015950206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:10:44.401264411Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=debug ts=2026-05-09T19:10:47.863211733Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-09T19:10:47.863234832Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-09T19:10:47.954446359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="238 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:10:47.964824105Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:10:49.015714006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:10:51.325189125Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-09T19:10:54.015185505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:10:54.786759807Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
ts=2026-05-09T19:10:55.607547338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T19:10:57.032284108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.8 MB" avg_comp="2.6 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:10:57.038536877Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:10:57.256195475Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:10:58.248120844Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-09T19:10:58.296363307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:10:58.300173347Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:10:59.015612885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:11:01.709422344Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-09T19:11:03.997523692Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:11:03.99792528Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:11:04.015427005Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:11:04.01545001Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:11:04.015459304Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:11:04.015457667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:11:05.170600681Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
ts=2026-05-09T19:11:07.113996418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:45052"
level=debug ts=2026-05-09T19:11:08.632201462Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-09T19:11:09.015674727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:11:12.093892933Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T19:11:14.015317219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:11:14.197242468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:34404"
level=debug ts=2026-05-09T19:11:15.555848599Z caller=checkpoint.go:372 msg="writing series" size="557 kB"
level=warn ts=2026-05-09T19:11:19.016130297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:11:19.017283042Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-09T19:11:19.017298495Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-05-09T19:11:19.110240041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:11:20.906191359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33018"
level=debug ts=2026-05-09T19:11:21.666913418Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:11:22.480214566Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-09T19:11:22.480236816Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=warn ts=2026-05-09T19:11:24.015167633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:11:25.608554274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:11:25.940343372Z caller=checkpoint.go:372 msg="writing series" size="314 kB"
level=warn ts=2026-05-09T19:11:29.015760349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:11:29.402277914Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-09T19:11:30.005860142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="238 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:11:30.046179343Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:11:32.863880069Z caller=checkpoint.go:372 msg="writing series" size="611 kB"
level=debug ts=2026-05-09T19:11:32.863936501Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-09T19:11:34.015719594Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:11:34.015751181Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T19:11:34.015744234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:11:34.015760831Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:11:36.324622873Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=warn ts=2026-05-09T19:11:39.015828713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:11:39.786683569Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=info ts=2026-05-09T19:11:42.244888784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:11:42.253934342Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:11:42.253963245Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:11:42.253997045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:11:42.260201015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:11:43.248915329Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-09T19:11:43.248936277Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-05-09T19:11:44.015230915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:11:46.709569592Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-09T19:11:49.016132336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:11:50.170687388Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
ts=2026-05-09T19:11:50.907788017Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47318"
level=debug ts=2026-05-09T19:11:51.134713769Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:11:53.632560558Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T19:11:54.015963326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:11:55.610481951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T19:11:55.903706217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:11:55.911734723Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:11:57.096896592Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T19:11:57.096921188Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=warn ts=2026-05-09T19:11:59.015912256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:00.555656367Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-09T19:12:01.508581912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:12:01.516440453Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:12:03.997515902Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:12:03.997801485Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:12:04.015382541Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:12:04.015401237Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:12:04.015410519Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:12:04.01541128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:04.01897531Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T19:12:04.018986788Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=debug ts=2026-05-09T19:12:07.478404618Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-09T19:12:09.015977389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:10.940467795Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=info ts=2026-05-09T19:12:12.101732962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="238 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:12:12.110162747Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:12:14.015971696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:14.401667521Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-09T19:12:15.40338132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:12:17.863148771Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
level=debug ts=2026-05-09T19:12:17.863172884Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-09T19:12:19.015707574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:21.117180056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:12:21.32482063Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-09T19:12:24.015773881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:24.786766954Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-09T19:12:25.613057283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T19:12:25.769031489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:12:25.777793852Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:12:28.248488172Z caller=checkpoint.go:372 msg="writing series" size="438 kB"
level=warn ts=2026-05-09T19:12:29.015651801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:31.709949285Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:12:34.015484367Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:12:34.01551316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:12:34.015520431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:12:34.015516471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:35.171398026Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T19:12:35.915607863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:12:35.920869371Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:12:38.632354421Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=warn ts=2026-05-09T19:12:39.0161565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:42.095206621Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-09T19:12:44.016185287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:12:44.207491635Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:60188"
level=debug ts=2026-05-09T19:12:45.555473343Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-09T19:12:45.959628923Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:12:49.015702624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:49.016835717Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=debug ts=2026-05-09T19:12:52.491676837Z caller=checkpoint.go:372 msg="writing series" size="9.2 MB"
level=debug ts=2026-05-09T19:12:52.491705644Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=10
level=warn ts=2026-05-09T19:12:54.015560954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:12:55.615585196Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:12:55.94087503Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
level=warn ts=2026-05-09T19:12:59.0156428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:12:59.401700963Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-09T19:13:02.863393272Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-09T19:13:03.998252666Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:13:03.999935925Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:13:04.015187293Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:13:04.015245025Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T19:13:04.015244446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:13:04.015273052Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:13:04.417485944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="261 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:13:04.427970006Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:13:06.324623619Z caller=checkpoint.go:372 msg="writing series" size="393 kB"
level=debug ts=2026-05-09T19:13:06.324648341Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-09T19:13:09.015768713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:13:09.786993504Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=debug ts=2026-05-09T19:13:13.24834656Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
level=debug ts=2026-05-09T19:13:13.440720774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:13:14.015583673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:13:14.21067529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:48420"
level=debug ts=2026-05-09T19:13:14.965595921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:13:14.967178334Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T19:13:16.709350513Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=info ts=2026-05-09T19:13:17.207005414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:13:17.211624969Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:13:19.015275144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:13:20.171412884Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T19:13:22.144066205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:13:22.15058307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:13:23.632894018Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T19:13:24.015174574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:13:25.617719586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:13:27.094699917Z caller=checkpoint.go:372 msg="writing series" size="596 kB"
level=debug ts=2026-05-09T19:13:27.094723315Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-09T19:13:29.015209786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:13:30.55589961Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-09T19:13:34.016067759Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:13:34.01611421Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:13:34.016123459Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:13:34.016122637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:13:34.017212887Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 kB" series=1
level=info ts=2026-05-09T19:13:34.164041908Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-09T19:13:34.166120566Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.149544115s
level=warn ts=2026-05-09T19:13:39.015924175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:13:44.015924177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:13:44.347559364Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:13:49.015611234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:13:54.01590771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:13:55.62066171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:13:56.161963121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="285 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:13:56.17215292Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:13:59.015977409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:13:59.724336737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:14:03.997516513Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:14:03.999485394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:14:04.016082186Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T19:14:04.016065268Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:14:04.016124171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:14:04.016134114Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:14:04.016140397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:14:04.016244513Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-09T19:14:04.016269274Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:14:04.016297233Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=20.713µs success=true
level=debug ts=2026-05-09T19:14:04.016559568Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
ts=2026-05-09T19:14:07.13153064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47840"
level=debug ts=2026-05-09T19:14:07.391651343Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T19:14:09.015388018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:14:10.76700528Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=debug ts=2026-05-09T19:14:11.997048355Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:14:14.015724222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:14:14.142245981Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=info ts=2026-05-09T19:14:16.574997781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:14:16.579728222Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:14:17.5174243Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T19:14:19.015752564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:14:20.732967624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:14:20.739371728Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:14:20.892295568Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:14:20.892323806Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-09T19:14:20.921970307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44042"
level=warn ts=2026-05-09T19:14:24.015236432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:14:24.267586323Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
ts=2026-05-09T19:14:25.623894094Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:14:27.642503894Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-09T19:14:29.015715624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:14:31.017692369Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T19:14:34.015131467Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:14:34.015157758Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:14:34.015165384Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:14:34.015201081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:14:34.392504012Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
ts=2026-05-09T19:14:37.134428246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:43964"
level=debug ts=2026-05-09T19:14:37.767416637Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T19:14:39.01570469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:14:40.782692444Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:14:40.782740519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:14:40.794144446Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:14:41.142437677Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:14:41.142460222Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-09T19:14:44.015881083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:14:44.51757446Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-09T19:14:47.89260595Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-09T19:14:49.016153852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:14:50.274996958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="285 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:14:50.28232543Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:14:51.1943749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:14:51.203017839Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:14:51.267243554Z caller=checkpoint.go:372 msg="writing series" size="356 kB"
level=warn ts=2026-05-09T19:14:54.015837715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:14:54.642260543Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-09T19:14:55.231031678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:14:55.626444719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:14:58.017481686Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-09T19:14:59.015369852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:14:59.063531676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:14:59.067244438Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:14:59.067265147Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:14:59.067286127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:14:59.07019453Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:15:01.393423902Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T19:15:01.393444838Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=6
level=info ts=2026-05-09T19:15:03.997815134Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:15:03.998467061Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-09T19:15:04.012067032Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T19:15:04.012144708Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975948 n=1
level=debug ts=2026-05-09T19:15:04.012188982Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975948 n=1
level=info ts=2026-05-09T19:15:04.01581478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:15:04.015836262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:15:04.015845316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:15:04.015881629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:15:04.767222948Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=debug ts=2026-05-09T19:15:07.858169041Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:15:07.858232345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:15:07.866285632Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:15:08.142351325Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-09T19:15:09.01582675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:15:11.547716797Z caller=checkpoint.go:372 msg="writing series" size="36 MB"
level=debug ts=2026-05-09T19:15:11.547741156Z caller=checkpoint.go:384 msg="flushing series" totalSize="36 MB" series=3
level=info ts=2026-05-09T19:15:11.805966639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:15:11.817401361Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:15:12.796221125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:15:12.800013482Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:15:14.015511169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:15:14.221531674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:37738"
level=debug ts=2026-05-09T19:15:14.891826414Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T19:15:18.267236177Z caller=checkpoint.go:372 msg="writing series" size="318 kB"
level=warn ts=2026-05-09T19:15:19.015339308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:15:20.92670584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49990"
level=debug ts=2026-05-09T19:15:21.642303931Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-09T19:15:24.015205191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:15:25.017724129Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
ts=2026-05-09T19:15:25.629657111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:15:26.193059485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:15:26.206200697Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:15:27.4485472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:15:27.456281431Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:15:28.392334977Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-09T19:15:29.015866835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:15:31.76702317Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:15:34.015431664Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:15:34.015458851Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:15:34.015467264Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:15:34.01553487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:15:34.320660311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:15:35.141840111Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-09T19:15:38.517495104Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T19:15:39.015753406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:15:40.431399114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="285 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:15:40.441677021Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:15:40.441699849Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:15:40.441731519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="285 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:15:40.449176265Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:15:41.891649078Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-05-09T19:15:41.891671935Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-09T19:15:44.01568844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:15:45.267294733Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-05-09T19:15:46.846035573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:15:46.851232171Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:15:48.643163153Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T19:15:48.643184519Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-09T19:15:49.016004353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:15:49.641744816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:15:52.016855404Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-09T19:15:54.015750049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:15:55.392519498Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
ts=2026-05-09T19:15:55.632337146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T19:15:56.704973815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="285 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:15:56.711812819Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:15:57.949154437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:15:57.955142038Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:15:58.7675606Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-09T19:15:59.016031363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:15:59.813889392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:15:59.820706437Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:16:02.141646127Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-09T19:16:03.997544611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:16:03.997974746Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:16:04.015202569Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:16:04.01522277Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:16:04.015230459Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:16:04.015256457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:16:05.389233832Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:16:05.516783119Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
ts=2026-05-09T19:16:07.140417296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:35208"
level=info ts=2026-05-09T19:16:07.693620498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:16:07.702022785Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:16:08.892461805Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=warn ts=2026-05-09T19:16:09.015740451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:16:12.267019713Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T19:16:14.015992751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:16:15.641920895Z caller=checkpoint.go:372 msg="writing series" size="517 kB"
level=warn ts=2026-05-09T19:16:19.015908885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:16:19.017172175Z caller=checkpoint.go:372 msg="writing series" size="345 kB"
level=debug ts=2026-05-09T19:16:19.017186389Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=9
level=debug ts=2026-05-09T19:16:22.394683583Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-09T19:16:22.39470538Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=1
level=warn ts=2026-05-09T19:16:24.015233688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:16:25.633897342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:16:25.767165395Z caller=checkpoint.go:372 msg="writing series" size="549 kB"
level=info ts=2026-05-09T19:16:28.316997636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="285 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:16:28.330857848Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:16:29.01526308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:16:29.142699749Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-09T19:16:32.5184253Z caller=checkpoint.go:372 msg="writing series" size="735 kB"
level=debug ts=2026-05-09T19:16:32.518448863Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-09T19:16:33.615114371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:16:33.618866331Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:16:34.015970341Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:16:34.015980302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:16:34.015998883Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:16:34.016007044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:16:35.892278645Z caller=checkpoint.go:372 msg="writing series" size="460 kB"
ts=2026-05-09T19:16:37.143953323Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:43592"
level=debug ts=2026-05-09T19:16:38.859189292Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:16:39.015566913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:16:39.267603621Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-09T19:16:41.89043695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:16:41.898691613Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:16:42.643163385Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T19:16:42.643185773Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=warn ts=2026-05-09T19:16:44.015998604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:16:44.227389782Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35350"
level=info ts=2026-05-09T19:16:44.81057762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:16:44.818021689Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:16:46.017487198Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T19:16:49.015592114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:16:49.392247455Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-09T19:16:50.272927053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:16:52.766964773Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T19:16:54.016041758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:16:55.636724713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:16:56.1429477Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-09T19:16:56.142970883Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=4
level=warn ts=2026-05-09T19:16:59.015743373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:16:59.517325429Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=info ts=2026-05-09T19:17:00.990122437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:17:01.001021609Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:17:02.895049876Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-09T19:17:02.895172956Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
level=info ts=2026-05-09T19:17:03.997884165Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:17:03.9984719Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:17:04.015815Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:17:04.015835461Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:17:04.015843828Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:17:04.015836483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:17:04.618750791Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:17:06.267633643Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-09T19:17:09.019180722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:17:09.641925961Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=debug ts=2026-05-09T19:17:13.01671112Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-09T19:17:13.179890486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="309 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:17:13.188325833Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:17:14.015528204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:17:16.392767696Z caller=checkpoint.go:372 msg="writing series" size="611 kB"
level=debug ts=2026-05-09T19:17:16.392789122Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-09T19:17:19.015864865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:17:19.767116593Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-09T19:17:23.142347647Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-09T19:17:24.016054474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:17:25.639451345Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:17:26.516903546Z caller=checkpoint.go:372 msg="writing series" size="322 kB"
level=info ts=2026-05-09T19:17:27.59569054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:17:27.602189845Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:17:29.015883551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:17:29.892570573Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-09T19:17:33.267368953Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T19:17:34.015510414Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:17:34.015536977Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T19:17:34.015526398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:17:34.015544859Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:17:34.106162455Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:17:34.336054161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:17:34.345553896Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:17:34.991981304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:17:34.999451306Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:17:36.642431822Z caller=checkpoint.go:372 msg="writing series" size="276 kB"
level=debug ts=2026-05-09T19:17:37.337143043Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:17:37.338741564Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T19:17:39.015407893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:17:40.0171248Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-09T19:17:43.392220394Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-09T19:17:44.015234606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:17:44.23426088Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:36596"
level=debug ts=2026-05-09T19:17:46.767462904Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T19:17:49.015370727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:17:50.142044928Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T19:17:53.029714971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:17:53.038947658Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:17:53.528741606Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-09T19:17:53.52880489Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=11
level=warn ts=2026-05-09T19:17:54.015364205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:17:55.640355735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:17:56.892308322Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=warn ts=2026-05-09T19:17:59.015674314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:18:00.267611902Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-09T19:18:03.641815616Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-09T19:18:03.997840712Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:18:03.998801915Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:18:04.016085254Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:18:04.016111606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:18:04.01612359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:18:04.016133472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:18:07.017673407Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=debug ts=2026-05-09T19:18:07.083853197Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:18:07.083926024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="413 kB" avg_comp="413 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:18:07.093038796Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T19:18:07.149921786Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:52590"
level=info ts=2026-05-09T19:18:08.554009857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:18:08.557602014Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:18:08.693784524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="309 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:18:08.702085079Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:18:09.016165725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:18:10.39183274Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=debug ts=2026-05-09T19:18:10.391862118Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-05-09T19:18:13.768680268Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=warn ts=2026-05-09T19:18:14.015453527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:18:14.238055414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:33848"
level=info ts=2026-05-09T19:18:16.471295211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:18:16.478191257Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:18:16.971410838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:18:16.977650182Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:18:16.977672758Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:18:16.977713609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:18:16.983385161Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:18:17.142130925Z caller=checkpoint.go:372 msg="writing series" size="642 B"
level=warn ts=2026-05-09T19:18:19.015682676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:18:20.51713739Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-09T19:18:22.95629186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:18:23.89195843Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T19:18:24.015379891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:18:25.642411004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:18:26.01651285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:18:26.020731656Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:18:27.267785795Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=debug ts=2026-05-09T19:18:27.267814351Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-05-09T19:18:29.016196933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:18:30.097186297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:18:30.111695864Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:18:30.642382983Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T19:18:34.016173348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:18:34.016212379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:18:34.016225988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:18:34.01623611Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:18:34.018159494Z caller=checkpoint.go:384 msg="flushing series" totalSize="61 kB" series=1
level=info ts=2026-05-09T19:18:34.028391111Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-09T19:18:34.03025402Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.013765025s
level=debug ts=2026-05-09T19:18:34.363706945Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T19:18:37.153717304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:36106"
level=warn ts=2026-05-09T19:18:39.015946466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:18:44.015316269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:18:47.564562098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:18:47.574166738Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=warn ts=2026-05-09T19:18:49.016280067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:18:52.993216387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:18:53.003218848Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:18:53.640387714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:18:53.64447075Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:18:54.015777062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:18:55.644656854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T19:18:59.015693664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:18:59.116645979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:18:59.177902655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:18:59.182027696Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:18:59.985037653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="309 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:19:00.012374687Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:19:03.997794175Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:19:04.000696205Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:19:04.015137907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:19:04.015173075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:19:04.015182832Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:19:04.015254461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:19:04.016274015Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T19:19:04.016396848Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-09T19:19:04.016405355Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:19:04.016428432Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.669µs success=true
level=debug ts=2026-05-09T19:19:04.016639234Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T19:19:06.348542046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:19:06.354649886Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:19:07.392222406Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T19:19:09.01598385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:19:09.20045461Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:19:10.76758768Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-09T19:19:14.015815012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:19:14.142359967Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=debug ts=2026-05-09T19:19:17.51680836Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T19:19:19.01616837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:19:20.892686985Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:19:20.892714171Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-09T19:19:20.950575095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46798"
level=warn ts=2026-05-09T19:19:24.015550175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:19:24.267731082Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
ts=2026-05-09T19:19:25.647471694Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:19:27.641865942Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-05-09T19:19:29.016022932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:19:31.017170541Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T19:19:32.239544815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:19:32.251043438Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:19:34.016010919Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:19:34.016016334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:19:34.016037956Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:19:34.016045218Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:19:34.392398807Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-09T19:19:35.964970091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:19:37.766761398Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T19:19:39.015434533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:19:41.142615438Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:19:41.142640583Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-09T19:19:41.970698074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:19:41.979499225Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:19:43.245969306Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:19:43.739393512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:19:43.742242937Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:19:44.015381893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:19:44.24580662Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:59184"
level=debug ts=2026-05-09T19:19:44.517065664Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-09T19:19:47.892469559Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-09T19:19:48.254261644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:19:48.258024914Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:19:49.016221444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:19:50.951982174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52344"
level=debug ts=2026-05-09T19:19:51.267544569Z caller=checkpoint.go:372 msg="writing series" size="490 kB"
level=warn ts=2026-05-09T19:19:54.015537575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:19:54.108945442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="333 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:19:54.117291523Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:19:54.64286496Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-09T19:19:55.65126333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:19:58.016990931Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-09T19:19:59.015633498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:01.393017729Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T19:20:01.393043399Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=6
level=info ts=2026-05-09T19:20:02.171126141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:20:02.178443044Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:20:02.759083677Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:20:03.997520169Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:20:03.997635723Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:20:04.01607975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:20:04.016074297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:04.01611811Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:20:04.016128963Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:20:04.767333657Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-09T19:20:08.142567555Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-09T19:20:09.016181106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:11.554449736Z caller=checkpoint.go:372 msg="writing series" size="42 MB"
level=debug ts=2026-05-09T19:20:11.554474499Z caller=checkpoint.go:384 msg="flushing series" totalSize="42 MB" series=3
level=warn ts=2026-05-09T19:20:14.015681291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:14.892236502Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T19:20:18.266871441Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-09T19:20:18.985111107Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:20:19.016223184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:21.642692272Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-09T19:20:24.015924294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:25.017214165Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
ts=2026-05-09T19:20:25.653208744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T19:20:26.170227305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:20:26.177983609Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:20:28.392028863Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-09T19:20:29.015586764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:31.767399787Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:20:34.01595662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:20:34.015971404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:34.01598631Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:20:34.015995316Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:20:35.141833915Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-09T19:20:37.033639115Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:20:38.516943845Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-09T19:20:38.540042019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:20:38.54770786Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:20:39.015879961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:20:40.795780707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:20:40.802120354Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:20:40.860148759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:20:40.863230164Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:20:41.892486383Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-09T19:20:44.016057687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:45.267404636Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=debug ts=2026-05-09T19:20:45.267428759Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-09T19:20:48.314777425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="333 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:20:48.324338097Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:20:48.643305007Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T19:20:48.643325229Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=warn ts=2026-05-09T19:20:49.015518366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:20:52.017151318Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-09T19:20:54.015780501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:20:54.139140221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:20:54.146560411Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:20:55.392306214Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-09T19:20:55.654492132Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:20:58.766868705Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-09T19:20:58.836706216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:20:58.838318002Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T19:20:59.015658944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:21:02.142631033Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-09T19:21:03.997541074Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:21:03.997563899Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=warn ts=2026-05-09T19:21:04.015980168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:21:04.01598846Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:21:04.016014358Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:21:04.016023243Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:21:05.516722345Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
ts=2026-05-09T19:21:07.164173532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:33130"
level=debug ts=2026-05-09T19:21:08.891806632Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-09T19:21:09.015986029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:21:11.193074116Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:21:12.26707641Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T19:21:14.015949005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:21:15.643367292Z caller=checkpoint.go:372 msg="writing series" size="654 kB"
level=debug ts=2026-05-09T19:21:15.643389947Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-09T19:21:19.015482564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:21:19.017710436Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-09T19:21:19.846277852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:21:19.853884876Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T19:21:20.959426772Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54440"
level=debug ts=2026-05-09T19:21:22.396466855Z caller=checkpoint.go:372 msg="writing series" size="4.9 MB"
level=debug ts=2026-05-09T19:21:22.396489564Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.1 MB" series=2
level=info ts=2026-05-09T19:21:23.082713394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:21:23.089973529Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:21:23.089992481Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:21:23.09001491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:21:23.096596912Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=warn ts=2026-05-09T19:21:24.016049137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:21:25.65634898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:21:25.767468852Z caller=checkpoint.go:372 msg="writing series" size="535 kB"
level=info ts=2026-05-09T19:21:28.097958119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:21:28.102638323Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:21:28.10266242Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:21:28.102684168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:21:28.105178551Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:21:29.016002986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:21:29.142464348Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-05-09T19:21:32.51803485Z caller=checkpoint.go:372 msg="writing series" size="820 kB"
level=debug ts=2026-05-09T19:21:32.518057476Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-09T19:21:34.015768657Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:21:34.015763573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:21:34.015791621Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:21:34.015800144Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:21:35.892651887Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=info ts=2026-05-09T19:21:36.00733577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:21:36.013893411Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-09T19:21:37.16687859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:48828"
level=warn ts=2026-05-09T19:21:39.016134285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:21:39.267195427Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-09T19:21:39.528918925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:21:39.536115298Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:21:40.492378699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:21:40.499539082Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:21:40.499558726Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:21:40.499581297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:21:40.506220558Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:21:42.644065211Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-09T19:21:42.644108939Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=3
level=warn ts=2026-05-09T19:21:44.015410781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:21:44.188864121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:21:44.192183342Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:21:45.404634882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:21:46.017428886Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-09T19:21:46.084669629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:21:46.553972855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="333 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:21:46.56145931Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:21:49.01596813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:21:49.391962087Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-09T19:21:52.76685594Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T19:21:54.016017785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:21:55.659510197Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:21:56.145278945Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-09T19:21:56.145301043Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=4
level=info ts=2026-05-09T19:21:57.231352946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:21:57.245664835Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:21:59.015998556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:21:59.517693887Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-09T19:22:02.893962307Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-09T19:22:02.893993756Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=2
level=info ts=2026-05-09T19:22:03.997599375Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:22:03.997657878Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:22:04.016039687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:22:04.016043141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:22:04.01606389Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:22:04.016074792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:22:06.26749785Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-09T19:22:06.636660548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:22:06.642611628Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T19:22:07.171593887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:41208"
level=warn ts=2026-05-09T19:22:09.016295953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:22:09.64190106Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=debug ts=2026-05-09T19:22:12.552145076Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:22:13.017334513Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-09T19:22:14.015238901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:22:16.394691037Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=debug ts=2026-05-09T19:22:16.394719334Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-09T19:22:16.646149333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:22:16.653598663Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-kszn\"}"
level=warn ts=2026-05-09T19:22:19.016319514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:22:19.162536647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:22:19.169856376Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:22:19.767669191Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-09T19:22:20.964757969Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38246"
level=info ts=2026-05-09T19:22:21.334395569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:22:21.338509399Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:22:23.141646389Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T19:22:24.016394626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:22:25.662635775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:22:26.517713123Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
level=warn ts=2026-05-09T19:22:29.018309366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:22:29.894167667Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-09T19:22:31.565636959Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:22:33.183708526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:22:33.196366077Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:22:33.267681289Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T19:22:34.015368071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:22:34.015397324Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:22:34.015405462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:22:34.015415603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:22:36.642405492Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-09T19:22:38.324539689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="358 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:22:38.332577018Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:22:38.332604184Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:22:38.332650584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="358 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:22:38.339570479Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:22:39.017355284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:22:40.017676352Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-09T19:22:40.616285585Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:22:43.398857737Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-09T19:22:44.015650172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:22:46.767702395Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T19:22:49.015311582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:22:50.141913473Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T19:22:53.089433126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="358 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:22:53.114607534Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:22:53.529930904Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T19:22:53.52995645Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=11
level=warn ts=2026-05-09T19:22:54.015346142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:22:55.435259384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:22:55.439519589Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T19:22:55.665941126Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:22:56.894575982Z caller=checkpoint.go:372 msg="writing series" size="748 kB"
level=warn ts=2026-05-09T19:22:59.015814613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:23:00.267181965Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-09T19:23:00.722595199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:23:00.73236198Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:23:01.00951121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:23:01.017221661Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:23:03.642509586Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-09T19:23:03.997728285Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:23:03.998186917Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:23:04.015548253Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:23:04.015589313Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T19:23:04.015602016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:23:04.015604096Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:23:06.21522638Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:23:07.017478168Z caller=checkpoint.go:372 msg="writing series" size="541 kB"
level=debug ts=2026-05-09T19:23:07.017500176Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
ts=2026-05-09T19:23:07.178140076Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:59286"
level=info ts=2026-05-09T19:23:08.78533508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:23:08.796238743Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:23:09.015686113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:23:10.39264219Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=debug ts=2026-05-09T19:23:13.767497855Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=warn ts=2026-05-09T19:23:14.015190675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:23:17.142592767Z caller=checkpoint.go:372 msg="writing series" size="642 B"
level=debug ts=2026-05-09T19:23:18.942432087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:23:19.015604982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:23:20.517264764Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-09T19:23:23.892597963Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T19:23:24.015202158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:23:25.618297464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="358 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:23:25.62546617Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T19:23:25.66885608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:23:27.267393551Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=debug ts=2026-05-09T19:23:27.267417802Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-09T19:23:27.915977293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:23:27.923219921Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:23:29.016243669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:23:30.642175543Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T19:23:34.0158088Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:23:34.015836056Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:23:34.015843916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:23:34.015871306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:23:34.016939105Z caller=checkpoint.go:384 msg="flushing series" totalSize="64 kB" series=1
level=info ts=2026-05-09T19:23:34.024357451Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-09T19:23:34.025909938Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009354555s
level=warn ts=2026-05-09T19:23:39.015732582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:23:41.897481192Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:23:44.015668598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:23:46.129052542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:23:46.139964804Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:23:48.039295699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:23:48.042723949Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:23:49.015512037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:23:50.976663539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59030"
level=warn ts=2026-05-09T19:23:54.015750353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:23:55.677430519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:23:56.8281541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:23:56.838177463Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-kszn\"}"
level=warn ts=2026-05-09T19:23:59.015454014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:24:01.004884444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:24:01.008773624Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:24:03.997417091Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:24:03.998468522Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-09T19:24:04.015871711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:24:04.015896156Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-09T19:24:04.015896594Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-09T19:24:04.015913615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:24:04.015921515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:24:04.016024558Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-09T19:24:04.016033256Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:24:04.0160499Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.793µs success=true
level=debug ts=2026-05-09T19:24:04.016278654Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=debug ts=2026-05-09T19:24:07.392177151Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-09T19:24:08.533572296Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:24:09.016271227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:24:09.914830416Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:24:10.767232882Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-05-09T19:24:13.199150902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:24:13.209148601Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:24:14.015217596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:24:14.141897308Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
ts=2026-05-09T19:24:14.280260642Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:53320"
level=debug ts=2026-05-09T19:24:17.517071496Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T19:24:19.01543654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:24:20.26954324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="358 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:24:20.278760564Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:24:20.892027206Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:24:20.892048861Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-09T19:24:20.977962941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35476"
level=warn ts=2026-05-09T19:24:24.016014037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:24:24.267161045Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
ts=2026-05-09T19:24:25.680396789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:24:27.641733254Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
level=warn ts=2026-05-09T19:24:29.016206014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:24:31.017347299Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T19:24:31.45737843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:24:31.46404485Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:24:31.464064285Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:24:31.46408703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:24:31.467017535Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:24:34.015409076Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:24:34.015432688Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:24:34.015440016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:24:34.015455331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:24:34.391935263Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-09T19:24:37.766905151Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T19:24:39.015975598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:24:41.141744316Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:24:41.141767865Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-05-09T19:24:43.265842335Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:24:43.684015237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:24:43.692837097Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:24:43.692858109Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:24:43.692882938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:24:43.698575027Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:24:44.015524629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:24:44.2839384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:40802"
level=info ts=2026-05-09T19:24:44.506893528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:24:44.510803186Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:24:44.516942344Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-09T19:24:47.891951711Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-09T19:24:48.495758459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:24:48.502550351Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-kszn\"}"
level=warn ts=2026-05-09T19:24:49.01597628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:24:51.267193299Z caller=checkpoint.go:372 msg="writing series" size="378 kB"
level=warn ts=2026-05-09T19:24:54.015630789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:24:54.641962228Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-09T19:24:55.685355039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:24:58.016725776Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=info ts=2026-05-09T19:24:58.926587866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.9 MB" avg_comp="2.2 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:24:58.934602186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:24:59.01561694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:25:00.277149874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:25:00.280439519Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:25:00.66624256Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:25:00.667980181Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T19:25:01.392040883Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T19:25:01.392060313Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=6
level=info ts=2026-05-09T19:25:03.997682888Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:25:03.99811785Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:25:04.015348654Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:25:04.015377383Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:25:04.01538735Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:25:04.015392774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:25:04.767298421Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=info ts=2026-05-09T19:25:07.083813266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="358 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:25:07.091307489Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:25:07.173401781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:25:07.18046411Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:25:08.142338507Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-09T19:25:09.015800582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:25:11.25087691Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:25:11.55408671Z caller=checkpoint.go:372 msg="writing series" size="44 MB"
level=debug ts=2026-05-09T19:25:11.554125223Z caller=checkpoint.go:384 msg="flushing series" totalSize="45 MB" series=3
level=warn ts=2026-05-09T19:25:14.015793064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:25:14.891414528Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T19:25:18.267312831Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=info ts=2026-05-09T19:25:18.545135116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:25:18.551226683Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:25:19.015384098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:25:20.982969586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39224"
level=debug ts=2026-05-09T19:25:21.641329059Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-09T19:25:24.015519897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:25:25.017350245Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
ts=2026-05-09T19:25:25.688088605Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:25:28.391582317Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-09T19:25:29.015616696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:25:31.756342141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:25:31.766338642Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=error ts=2026-05-09T19:25:31.769298928Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:25:34.015283379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:25:34.015308792Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:25:34.015325935Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:25:34.015318062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:25:35.14237249Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-09T19:25:37.243921134Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:25:38.516447223Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T19:25:39.015529865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:25:39.829346605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:25:39.837426975Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:25:41.891527011Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-09T19:25:41.891549949Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-09T19:25:42.568315103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:25:42.572041691Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:25:44.016078352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:25:44.291749608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49060"
level=debug ts=2026-05-09T19:25:45.266341932Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-09T19:25:48.643392254Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T19:25:48.643413327Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-09T19:25:49.016129641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:25:49.328366819Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:25:52.016461322Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-09T19:25:54.015160303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:25:55.392329292Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T19:25:55.514524925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="358 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:25:55.523269453Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T19:25:55.690009382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:25:58.766314872Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-09T19:25:59.015646835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:02.14201876Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-09T19:26:03.798731423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:26:03.806528926Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:26:03.997534373Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:26:03.998345404Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=warn ts=2026-05-09T19:26:04.015745529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:26:04.015760609Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:26:04.015788584Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:26:04.015803137Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:26:05.517226869Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-09T19:26:08.892336297Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-09T19:26:09.015554557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:11.068921446Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:26:12.26657832Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-09T19:26:13.372588723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:26:13.376720444Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:26:14.015736506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:15.641857636Z caller=checkpoint.go:372 msg="writing series" size="680 kB"
level=debug ts=2026-05-09T19:26:15.641879605Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-09T19:26:19.015772576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:19.016977352Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
ts=2026-05-09T19:26:20.987612578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48670"
level=debug ts=2026-05-09T19:26:22.400895633Z caller=checkpoint.go:372 msg="writing series" size="8.1 MB"
level=debug ts=2026-05-09T19:26:22.401012051Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.3 MB" series=2
level=warn ts=2026-05-09T19:26:24.01613933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:24.60480301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:26:25.508214847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:26:25.51671938Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-09T19:26:25.692489633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:26:25.767490287Z caller=checkpoint.go:372 msg="writing series" size="476 kB"
level=warn ts=2026-05-09T19:26:29.01597458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:26:29.097288532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:26:29.104743925Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:26:29.141904723Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-09T19:26:32.517383175Z caller=checkpoint.go:372 msg="writing series" size="955 kB"
level=debug ts=2026-05-09T19:26:32.517404353Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-09T19:26:34.015895303Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:26:34.015928677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:26:34.015939001Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:26:34.015960284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:35.891964807Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
level=warn ts=2026-05-09T19:26:39.015521316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:39.266701574Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:26:40.406004229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:26:40.413513407Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:26:42.051982463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:26:42.059154669Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:26:42.643479023Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-09T19:26:42.643499128Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=warn ts=2026-05-09T19:26:44.016023515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:44.415747049Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:26:46.016466943Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-09T19:26:47.085425521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:26:47.091876833Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:26:49.015213099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:49.392156102Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-09T19:26:52.767247229Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T19:26:54.016269575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:26:55.695818778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:26:55.811860803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:26:55.816089309Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:26:56.14452611Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-09T19:26:56.14455194Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=4
level=warn ts=2026-05-09T19:26:59.015489353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:26:59.516663382Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-05-09T19:27:02.892962547Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-09T19:27:02.892984177Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=info ts=2026-05-09T19:27:03.997512994Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:27:03.997879864Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:27:04.015126071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:27:04.015141574Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:27:04.015148821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:27:04.015197251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:27:06.267310848Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
ts=2026-05-09T19:27:07.202439309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:41974"
level=warn ts=2026-05-09T19:27:09.015871466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:27:09.641713015Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=debug ts=2026-05-09T19:27:09.79929113Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:27:12.010362536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:27:12.021309776Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:27:13.016776497Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-05-09T19:27:14.015486499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:27:14.550406782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:27:15.938000369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:27:15.945828006Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:27:16.391857622Z caller=checkpoint.go:372 msg="writing series" size="640 kB"
level=debug ts=2026-05-09T19:27:16.391875637Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-09T19:27:19.016006172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:27:19.767179012Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-09T19:27:23.141908027Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-09T19:27:24.015528352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:27:25.699619112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:27:26.516519075Z caller=checkpoint.go:372 msg="writing series" size="366 kB"
level=warn ts=2026-05-09T19:27:29.016043355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:27:29.891542396Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-09T19:27:33.267181816Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T19:27:34.015573253Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:27:34.015602912Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:27:34.015610498Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:27:34.015606135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:27:35.745894351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:27:35.750569719Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:27:36.552216483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:27:36.559941645Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:27:36.642395984Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-09T19:27:37.205233333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:38202"
level=warn ts=2026-05-09T19:27:39.01620629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:27:39.805018565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:27:39.812084273Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:27:40.017016791Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-09T19:27:43.391440064Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-09T19:27:44.015199084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:27:44.125623462Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T19:27:44.303084218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:44904"
level=debug ts=2026-05-09T19:27:46.766576171Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T19:27:49.01530212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:27:50.141537589Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T19:27:53.381558511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:27:53.389643405Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:27:53.527753627Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-09T19:27:53.527774504Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=11
level=warn ts=2026-05-09T19:27:54.015247158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:27:55.704241836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T19:27:56.876305983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:27:56.883227389Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:27:56.883250647Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:27:56.883273554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:27:56.889302849Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:27:56.893557526Z caller=checkpoint.go:372 msg="writing series" size="866 kB"
level=warn ts=2026-05-09T19:27:59.015733267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:28:00.266475797Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-09T19:28:00.545029525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:28:03.641324933Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-09T19:28:03.99778601Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:28:03.997796962Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:28:04.015361479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:28:04.01535883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:28:04.015403292Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:28:04.015411639Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:28:05.301172465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:28:05.314211046Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:28:07.016634475Z caller=checkpoint.go:372 msg="writing series" size="500 kB"
level=debug ts=2026-05-09T19:28:07.016655795Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-09T19:28:09.015258989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:28:10.100191569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:28:10.117608111Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:28:10.391637169Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-09T19:28:13.767385209Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=warn ts=2026-05-09T19:28:14.015254327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:28:14.311511683Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:57024"
level=debug ts=2026-05-09T19:28:14.982236426Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:28:14.982290393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:28:14.987962803Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:28:17.142008224Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=warn ts=2026-05-09T19:28:19.01562455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:28:20.4202973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:28:20.424270796Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:28:20.424293053Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:28:20.424314996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:28:20.427229753Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:28:20.516533656Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-09T19:28:21.006381226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:58140"
level=info ts=2026-05-09T19:28:21.427290677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:28:21.433923453Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:28:21.433949011Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:28:21.433985605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:28:21.440707069Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:28:23.893202908Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T19:28:24.015940566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:28:25.711447224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T19:28:25.838060952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:28:25.846636456Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:28:27.266671244Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=debug ts=2026-05-09T19:28:27.266695204Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-09T19:28:27.452323249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:28:27.458019415Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T19:28:29.01637009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:28:30.64133921Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-09T19:28:32.647322613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:28:32.654063928Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:28:32.773328764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:28:32.776702447Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:28:34.015551279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:28:34.015581112Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:28:34.015591863Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:28:34.015655135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:28:34.016784248Z caller=checkpoint.go:384 msg="flushing series" totalSize="70 kB" series=1
level=info ts=2026-05-09T19:28:34.024078848Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-09T19:28:34.025603307Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009401176s
level=warn ts=2026-05-09T19:28:39.015354145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:28:42.086493324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:28:42.103909822Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:28:43.631327938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:28:43.635530296Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:28:44.015330476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:28:46.390590521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:28:46.403434586Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:28:49.015593521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:28:49.984334359Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:28:53.725523025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:28:53.729020268Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:28:54.015988194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:28:54.450156956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:28:54.456585744Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T19:28:55.714661632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=warn ts=2026-05-09T19:28:59.016325315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:28:59.22121099Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:28:59.223088054Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T19:29:02.003588788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:29:02.019501018Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:29:03.998805972Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:29:04.00000229Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:29:04.015651691Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-09T19:29:04.015697764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:29:04.01572633Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:29:04.015737824Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:29:04.015747702Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:29:04.015778281Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-09T19:29:04.015786287Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:29:04.015806579Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.533µs success=true
level=debug ts=2026-05-09T19:29:04.016016956Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
ts=2026-05-09T19:29:07.215282819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:48468"
level=debug ts=2026-05-09T19:29:07.394882648Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T19:29:09.016561845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:29:10.746540957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:29:10.75443293Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:29:10.754457182Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:29:10.754493985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:29:10.761459152Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:29:10.766857942Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-09T19:29:14.015411545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:29:14.142265996Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=debug ts=2026-05-09T19:29:17.51624762Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T19:29:19.015246241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:29:20.891889015Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:29:20.891919487Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-09T19:29:23.458475037Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:29:24.015404103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:29:24.267121495Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-09T19:29:24.745315298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:29:24.756691634Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T19:29:25.719387586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:29:27.641377663Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-09T19:29:27.752724927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:29:27.760070093Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:29:29.015326726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:29:31.016395409Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T19:29:34.01531896Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:29:34.015331066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:29:34.015355425Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:29:34.015363123Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:29:34.396720662Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T19:29:36.280340621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:29:36.28848827Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:29:36.750566482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:29:36.75442059Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:29:37.469356539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:29:37.475256769Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:29:37.766807463Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T19:29:39.015263569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:29:41.142195453Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:29:41.14221758Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-09T19:29:44.015468664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:29:44.516064098Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-09T19:29:46.057314629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:29:46.065028492Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:29:47.218117324Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:29:47.891794677Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-09T19:29:49.015558699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:29:51.017630993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42874"
level=debug ts=2026-05-09T19:29:51.266614659Z caller=checkpoint.go:372 msg="writing series" size="481 kB"
level=debug ts=2026-05-09T19:29:51.834379042Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:29:52.23330589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:29:52.236871132Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:29:53.40216592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:29:53.419988192Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:29:54.016242177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:29:54.641476048Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-09T19:29:55.721839757Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:29:58.017024859Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=warn ts=2026-05-09T19:29:59.015897065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:30:01.394555546Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-09T19:30:01.394585918Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=6
level=info ts=2026-05-09T19:30:03.998137865Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:30:03.999887071Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T19:30:03.999952471Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975949 n=1
level=debug ts=2026-05-09T19:30:04.00000828Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975949 n=1
level=debug ts=2026-05-09T19:30:04.016051266Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:30:04.016150015Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:30:04.016173819Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:30:04.016181469Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:30:04.016188768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:30:04.767140461Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-09T19:30:08.141655357Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-09T19:30:09.015897083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:30:11.556802673Z caller=checkpoint.go:372 msg="writing series" size="48 MB"
level=debug ts=2026-05-09T19:30:11.556828595Z caller=checkpoint.go:384 msg="flushing series" totalSize="48 MB" series=3
level=warn ts=2026-05-09T19:30:14.015381035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:30:14.898263338Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T19:30:18.266603902Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=warn ts=2026-05-09T19:30:19.015708634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:30:21.019061168Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:44722"
level=debug ts=2026-05-09T19:30:21.641023184Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-09T19:30:22.425370459Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:30:22.794673471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:30:22.80471487Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T19:30:23.249109152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:30:23.257428772Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=warn ts=2026-05-09T19:30:24.015513753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:30:25.016520171Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-09T19:30:25.722679979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T19:30:27.064947191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:30:27.075715677Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:30:28.391306624Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=warn ts=2026-05-09T19:30:29.015592682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:30:29.551589869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:30:29.555718689Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:30:31.766691513Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:30:31.827136092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:30:31.834393541Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:30:34.015542406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:30:34.015578119Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:30:34.015585866Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:30:34.015648813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:30:35.127056216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:30:35.141356851Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=info ts=2026-05-09T19:30:37.384969872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:30:37.388231973Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:30:38.516422404Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-09T19:30:39.015909759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:30:41.891803533Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T19:30:44.015408779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:30:44.324843702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49596"
level=info ts=2026-05-09T19:30:44.603444387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:30:44.610159337Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:30:45.267152435Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-09T19:30:45.267172516Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=debug ts=2026-05-09T19:30:48.642700206Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T19:30:48.642720448Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-09T19:30:49.015758077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:30:52.016927642Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-09T19:30:54.016045279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:30:55.392161153Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-09T19:30:55.725183585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:30:57.034369774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:30:58.766985057Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-09T19:30:59.015707055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:31:02.141772849Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-09T19:31:03.997603428Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:31:03.998404749Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=warn ts=2026-05-09T19:31:04.015727097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:31:04.015740051Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:31:04.015763841Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:31:04.01577237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:31:05.516740021Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T19:31:06.940636125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:31:06.947155203Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:31:08.891676219Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=warn ts=2026-05-09T19:31:09.015790063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:31:09.288199284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:31:09.295394479Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:31:09.295414813Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:31:09.29543942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:31:09.301670033Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:31:09.619812489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:31:09.623339762Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:31:09.623359692Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:31:09.62339312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:31:09.62633479Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:31:12.266778768Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T19:31:14.015448418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:31:14.327846182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:42450"
level=debug ts=2026-05-09T19:31:15.642619884Z caller=checkpoint.go:372 msg="writing series" size="831 kB"
level=debug ts=2026-05-09T19:31:15.642642937Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-05-09T19:31:16.216290098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:31:16.223992232Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:31:19.015693836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:31:19.016885757Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=debug ts=2026-05-09T19:31:20.284255464Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:31:20.634389257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:31:20.642253724Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:31:20.740472039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:31:20.743798726Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:31:22.400693704Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T19:31:22.400714533Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=info ts=2026-05-09T19:31:22.779686895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:31:22.786989381Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:31:24.015182127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:31:25.72810693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:31:25.767212024Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
level=debug ts=2026-05-09T19:31:27.650469821Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:31:29.01623013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:31:29.141606457Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=info ts=2026-05-09T19:31:29.486939548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:31:29.490294682Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:31:32.516921099Z caller=checkpoint.go:372 msg="writing series" size="904 kB"
level=debug ts=2026-05-09T19:31:32.516949205Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-09T19:31:34.015206074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:31:34.015206787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:31:34.015233874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:31:34.015244442Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:31:35.89217738Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
level=info ts=2026-05-09T19:31:37.787471467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:31:37.794734408Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:31:39.015486688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:31:39.266509687Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-09T19:31:41.11588251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:31:41.119923768Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:31:42.64293609Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-09T19:31:42.642957312Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=3
level=warn ts=2026-05-09T19:31:44.015894891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:31:46.016487319Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-09T19:31:47.776313075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:31:47.783525442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:31:49.015710258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:31:49.391714144Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=debug ts=2026-05-09T19:31:52.766050627Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T19:31:54.016068246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:31:55.418707163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T19:31:55.731689221Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:31:56.144977339Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-09T19:31:56.144998257Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=4
level=warn ts=2026-05-09T19:31:59.015991489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:31:59.516382641Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-09T19:32:02.597748692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:32:02.605119702Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:32:02.892800827Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-09T19:32:02.892824739Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=debug ts=2026-05-09T19:32:03.87571172Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:32:03.997639103Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:32:03.997641948Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:32:04.016005049Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:32:04.015999031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:32:04.016022522Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:32:04.016030078Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:32:06.266108966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:32:06.266202339Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=error ts=2026-05-09T19:32:06.273820194Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
ts=2026-05-09T19:32:07.231306124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:34340"
level=info ts=2026-05-09T19:32:08.30349182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="382 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:32:08.312548242Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:32:09.015539248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:32:09.642080228Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=debug ts=2026-05-09T19:32:13.016964739Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-09T19:32:14.015544457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:32:16.392361745Z caller=checkpoint.go:372 msg="writing series" size="640 kB"
level=debug ts=2026-05-09T19:32:16.392390961Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-09T19:32:19.015547291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:32:19.766460617Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-09T19:32:20.925031435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:32:20.931265477Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:32:23.141286538Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-09T19:32:24.015558013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:32:24.871911823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:32:24.878332374Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-09T19:32:25.735417633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:32:26.517214364Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=warn ts=2026-05-09T19:32:29.015186359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:32:29.518378741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:32:29.891399912Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:32:32.040317409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:32:32.044712245Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:32:33.266984019Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T19:32:34.016021484Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:32:34.016046359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:32:34.016054166Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:32:34.016054808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:32:36.641503617Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-05-09T19:32:36.687478104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:32:36.694131614Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-09T19:32:37.234249726Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:53160"
level=warn ts=2026-05-09T19:32:39.01608286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:32:40.016877601Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-09T19:32:43.391300473Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-09T19:32:44.015255682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:32:46.766357614Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=debug ts=2026-05-09T19:32:47.072455666Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:32:47.074245107Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T19:32:49.015793075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:32:49.687930772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:32:49.69711607Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:32:50.083662173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="406 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:32:50.090496979Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:32:50.141762662Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-09T19:32:50.141784959Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-09T19:32:50.338149167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:32:50.343676079Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:32:53.532079613Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T19:32:53.532125618Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=1
level=warn ts=2026-05-09T19:32:54.015936801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:32:55.565327461Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T19:32:55.739133331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:32:56.892360211Z caller=checkpoint.go:372 msg="writing series" size="768 kB"
level=warn ts=2026-05-09T19:32:59.01566974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:33:00.266266155Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-09T19:33:03.641508157Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-09T19:33:03.998374666Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:33:03.999327447Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:33:04.015546407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:33:04.015563304Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:33:04.015570685Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:33:04.01558596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:33:05.961750133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:33:05.97006555Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:33:07.016887609Z caller=checkpoint.go:372 msg="writing series" size="684 kB"
level=debug ts=2026-05-09T19:33:07.016912135Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-09T19:33:09.015834239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:33:09.481987152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:33:09.485957141Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:33:10.392135325Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-09T19:33:13.767204419Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=warn ts=2026-05-09T19:33:14.015891883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:33:14.027054201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:33:14.030840122Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:33:17.141593521Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T19:33:17.438542319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:33:17.447978489Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:33:19.016159211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:33:20.516211876Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-09T19:33:23.891637975Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T19:33:24.015653932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:33:25.742070992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:33:26.257127026Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:33:27.267291902Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=debug ts=2026-05-09T19:33:27.267315269Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=warn ts=2026-05-09T19:33:29.015332355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:33:30.641108729Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-09T19:33:31.517488119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:33:31.528312312Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:33:34.015955871Z caller=checkpoint.go:384 msg="flushing series" totalSize="75 kB" series=1
level=warn ts=2026-05-09T19:33:34.015998359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:33:34.015996953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:33:34.016021941Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:33:34.016030924Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:33:34.026109945Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-09T19:33:34.027620772Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011692125s
level=warn ts=2026-05-09T19:33:39.015946384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:33:39.24994282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:33:39.256384903Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T19:33:39.391209634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="406 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:33:39.398503103Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:33:44.015287502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:33:44.980418565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:33:49.015963843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:33:54.01602741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:33:55.273749643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:33:55.278543004Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T19:33:55.745031487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:33:55.821986187Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:33:55.822044518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="850 kB" avg_comp="850 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:33:55.826578804Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T19:33:57.411876388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:33:57.418360889Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:33:57.746446213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:33:57.752950763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:33:59.015828555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:34:03.997517301Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:34:03.997993431Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:34:04.015272352Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:34:04.015292671Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:34:04.015300111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:34:04.015299033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:34:04.016379057Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T19:34:04.016504495Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-09T19:34:04.016510864Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:34:04.016528125Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.403µs success=true
level=debug ts=2026-05-09T19:34:04.016728957Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-09T19:34:06.493203544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:34:06.497074846Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:34:07.392724993Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T19:34:09.016013682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:34:09.327142367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="866 kB" avg_comp="866 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:34:09.332695429Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T19:34:10.766894842Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-05-09T19:34:14.015606223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:34:14.142435392Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
ts=2026-05-09T19:34:14.343461485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:38362"
level=debug ts=2026-05-09T19:34:17.516938526Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T19:34:19.015991111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:34:20.892706423Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:34:20.892729669Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T19:34:22.786781247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:34:22.797589833Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:34:22.797610461Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:34:22.797637452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:34:22.804920004Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=warn ts=2026-05-09T19:34:24.015438434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:34:24.267069068Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-09T19:34:25.747562436Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:34:26.017313146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:34:27.64285037Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=debug ts=2026-05-09T19:34:28.218613273Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:34:29.016172361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:34:31.000670357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="406 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:34:31.007721592Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:34:31.016890863Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T19:34:34.015140866Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:34:34.015186616Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:34:34.015199593Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:34:34.015200537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:34:34.392217942Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-09T19:34:37.243201731Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:51904"
level=debug ts=2026-05-09T19:34:37.766920035Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-09T19:34:37.897473404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:34:37.901671402Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:34:37.909708306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:34:37.914425354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:34:37.916204792Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:34:37.919369497Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:34:37.919386747Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:34:37.919410117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:34:37.924251426Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T19:34:39.015938516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:34:39.239846393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="866 kB" avg_comp="866 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:34:39.243963064Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T19:34:41.142414289Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:34:41.142434392Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-09T19:34:44.01571568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:34:44.517390742Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-09T19:34:45.882475878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:34:45.889156214Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:34:45.889176048Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:34:45.889200433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:34:45.895033984Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:34:47.892694745Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-09T19:34:49.016193464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:34:51.267158051Z caller=checkpoint.go:372 msg="writing series" size="417 kB"
level=info ts=2026-05-09T19:34:52.962390651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:34:52.968421673Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:34:52.968444194Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:34:52.968466294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:34:52.971241326Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:34:54.015936811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:34:54.642739323Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-09T19:34:54.725904624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:34:54.732037799Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T19:34:55.749681432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:34:56.139579744Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:34:56.798636777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:34:56.806700527Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:34:58.017275322Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-09T19:34:59.01597944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:35:00.582707191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:35:01.393043715Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-09T19:35:01.393064811Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=6
level=info ts=2026-05-09T19:35:01.984632339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:35:01.990578603Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:35:03.351406451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:35:03.358299448Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:35:03.997563175Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:35:03.997812718Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:35:04.015212574Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:35:04.015230977Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:35:04.015238409Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:35:04.015258581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:35:04.767085943Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-09T19:35:07.152244105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:35:07.15582369Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T19:35:07.246436149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47228"
level=debug ts=2026-05-09T19:35:08.142587928Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-09T19:35:09.015535146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:35:11.576350662Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-09T19:35:11.576380228Z caller=checkpoint.go:384 msg="flushing series" totalSize="54 MB" series=3
level=warn ts=2026-05-09T19:35:14.015547662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:35:14.349385387Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:56496"
level=info ts=2026-05-09T19:35:14.743402868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="430 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:35:14.751192714Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:35:14.751214929Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:35:14.751255821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="430 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:35:14.758261424Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:35:14.892698262Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-09T19:35:18.267381298Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-09T19:35:19.016019752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:35:21.049504712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45814"
level=debug ts=2026-05-09T19:35:21.642398102Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-09T19:35:22.877303558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:35:22.884927646Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:35:24.016059424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:35:25.017329363Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-09T19:35:25.289440643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:35:25.311567459Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T19:35:25.751886994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:35:26.882317332Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:35:28.392533876Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=warn ts=2026-05-09T19:35:29.015644614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:35:30.844245988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="430 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:35:30.852884358Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:35:31.392542209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="866 kB" avg_comp="866 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:35:31.396779125Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T19:35:31.767663927Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-09T19:35:34.015580733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:35:34.015614125Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:35:34.015631882Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:35:34.015639049Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:35:34.980060455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:35:34.983692905Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:35:35.142911355Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=debug ts=2026-05-09T19:35:38.517829117Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-05-09T19:35:39.015661294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:35:41.687564992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:35:41.691505367Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:35:41.892185076Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-09T19:35:41.892205728Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-09T19:35:44.015802437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:35:44.913846231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:35:45.267577992Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-09T19:35:48.089661239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:35:48.105377917Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:35:48.643781248Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T19:35:48.643802591Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=warn ts=2026-05-09T19:35:49.015673468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:35:52.017521064Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-09T19:35:54.015595935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:35:55.392764434Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-09T19:35:55.753776751Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:35:58.767698974Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-05-09T19:35:59.015717257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:35:59.065967308Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:36:02.142878765Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-09T19:36:02.360330759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:36:02.37230071Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:36:03.997697234Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:36:03.997719759Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:36:04.01604283Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:36:04.016060619Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:36:04.016068746Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:36:04.016076258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:36:04.135647866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="430 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:36:04.143914332Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:36:05.517179147Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-09T19:36:08.892799861Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=warn ts=2026-05-09T19:36:09.01584938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:36:12.267069655Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-09T19:36:14.015266059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:36:14.354199956Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:55720"
level=debug ts=2026-05-09T19:36:15.642604894Z caller=checkpoint.go:372 msg="writing series" size="777 kB"
level=debug ts=2026-05-09T19:36:15.642627442Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=info ts=2026-05-09T19:36:18.420717208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:36:18.430820939Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:36:19.015697066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:36:19.016994159Z caller=checkpoint.go:372 msg="writing series" size="377 kB"
level=info ts=2026-05-09T19:36:22.247378366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:36:22.254139945Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:36:22.402183355Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T19:36:22.402217657Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=warn ts=2026-05-09T19:36:24.017237482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:36:25.756690405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:36:25.767657796Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
level=info ts=2026-05-09T19:36:25.935026285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="866 kB" avg_comp="866 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:36:25.940633393Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-09T19:36:28.393659145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:36:28.400543711Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:36:29.015281236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:36:29.142817606Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-09T19:36:31.600154637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:36:31.604657387Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:36:32.441256676Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:36:32.518057599Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-09T19:36:32.518079424Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-09T19:36:34.015296421Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:36:34.015395453Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T19:36:34.015386133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:36:34.015408565Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:36:35.892905164Z caller=checkpoint.go:372 msg="writing series" size="482 kB"
level=debug ts=2026-05-09T19:36:37.275950587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:36:37.277917848Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T19:36:39.015345173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:36:39.267561507Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-09T19:36:40.469177697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:36:40.473536503Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:36:42.645616269Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T19:36:42.64564375Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=3
level=warn ts=2026-05-09T19:36:44.015222624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:36:44.358066657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49118"
level=debug ts=2026-05-09T19:36:46.017317963Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T19:36:49.015699833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:36:49.392735819Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
level=info ts=2026-05-09T19:36:51.654385497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:36:51.665807517Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:36:52.767443587Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-09T19:36:54.015906703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:36:54.322374452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:36:54.33163415Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
ts=2026-05-09T19:36:55.761139135Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:36:56.145200638Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-09T19:36:56.145222897Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=4
level=warn ts=2026-05-09T19:36:59.01579669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:36:59.517838997Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=debug ts=2026-05-09T19:37:02.893549441Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-09T19:37:02.8935773Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=info ts=2026-05-09T19:37:02.936089059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:37:02.94306421Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:37:03.997499752Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:37:03.997536354Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:37:04.015947049Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:37:04.015980808Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T19:37:04.015971122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:37:04.015990986Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:37:06.26718655Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
ts=2026-05-09T19:37:07.253187987Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:55796"
level=debug ts=2026-05-09T19:37:08.172500045Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:37:09.015764303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:37:09.642602309Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=debug ts=2026-05-09T19:37:13.017401614Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-09T19:37:13.68713173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:37:13.696257245Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:37:14.015164069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:37:15.472823296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="865 kB" avg_comp="865 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:37:15.478234596Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T19:37:16.392014876Z caller=checkpoint.go:372 msg="writing series" size="691 kB"
level=debug ts=2026-05-09T19:37:16.392036972Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-09T19:37:19.015853077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:37:19.767267147Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-09T19:37:22.76841458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:37:23.142745464Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-09T19:37:24.015696835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:37:24.992583258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:37:25.000212953Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:37:25.000237001Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:37:25.000264306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:37:25.007707215Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-09T19:37:25.767616144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:37:26.517584999Z caller=checkpoint.go:372 msg="writing series" size="390 kB"
level=warn ts=2026-05-09T19:37:29.01554102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:37:29.838123711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:37:29.843902728Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:37:29.843927791Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:37:29.843949036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:37:29.847391372Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:37:29.892691095Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-09T19:37:33.267412105Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T19:37:34.015532674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:37:34.015575792Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:37:34.015586089Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:37:34.015627433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:37:34.500289892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:37:34.508244803Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:37:36.642437675Z caller=checkpoint.go:372 msg="writing series" size="307 kB"
level=info ts=2026-05-09T19:37:37.382126745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:37:37.388942491Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:37:39.016205993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:37:40.017318265Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-09T19:37:40.089545316Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:37:40.17762203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:37:40.185735923Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:37:43.392596033Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-09T19:37:44.015656904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:37:46.767385713Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=info ts=2026-05-09T19:37:48.028111463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:37:48.032383123Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:37:49.015987836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:37:49.70341526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:37:49.710147211Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:37:50.14222743Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-09T19:37:50.142256896Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
ts=2026-05-09T19:37:51.063814746Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46806"
level=info ts=2026-05-09T19:37:52.127682228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:37:52.135995588Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=debug ts=2026-05-09T19:37:53.530232569Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-09T19:37:53.530255827Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=1
level=warn ts=2026-05-09T19:37:54.01592519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:37:55.769444811Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T19:37:56.169396771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="866 kB" avg_comp="866 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:37:56.174053619Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T19:37:56.892734283Z caller=checkpoint.go:372 msg="writing series" size="931 kB"
level=warn ts=2026-05-09T19:37:59.015535546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:38:00.266949096Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-09T19:38:03.383567969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:38:03.390697371Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:38:03.642460261Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-09T19:38:03.997591525Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:38:03.998349928Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:38:04.015710114Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:38:04.015730142Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:38:04.015740523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:38:04.015746551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:38:06.716438304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:38:06.723517923Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:38:07.017850044Z caller=checkpoint.go:372 msg="writing series" size="573 kB"
level=debug ts=2026-05-09T19:38:07.017872278Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
ts=2026-05-09T19:38:07.257247615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:37462"
level=warn ts=2026-05-09T19:38:09.015971728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:38:09.15239603Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:38:10.392202419Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-05-09T19:38:13.767927418Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=warn ts=2026-05-09T19:38:14.015639946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:38:14.365637863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:52184"
level=info ts=2026-05-09T19:38:15.569017105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:38:15.572738296Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:38:17.142551895Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=debug ts=2026-05-09T19:38:17.565523763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:38:19.015916584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:38:19.048072966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:38:19.054565454Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:38:20.516901749Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=debug ts=2026-05-09T19:38:23.891850969Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-09T19:38:24.015164183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:38:25.77214822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:38:27.267277174Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=debug ts=2026-05-09T19:38:27.267300476Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=warn ts=2026-05-09T19:38:29.015864939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:38:30.64270861Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-09T19:38:32.933435349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:38:32.940572818Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:38:34.016040917Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:38:34.016067341Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:38:34.016076333Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:38:34.016082754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:38:34.017115231Z caller=checkpoint.go:384 msg="flushing series" totalSize="78 kB" series=1
level=info ts=2026-05-09T19:38:34.022572516Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-09T19:38:34.024110503Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007456199s
level=info ts=2026-05-09T19:38:37.384397867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="866 kB" avg_comp="866 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:38:37.389000805Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-09T19:38:39.015960907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:38:43.273762936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:38:43.281916671Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:38:43.560753217Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:38:44.015744714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:38:47.548964832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:38:47.559401436Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=warn ts=2026-05-09T19:38:49.016082045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:38:50.400235336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:38:50.408811531Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:38:51.180886206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:38:51.187075959Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:38:54.01571619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:38:55.775278913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=warn ts=2026-05-09T19:38:59.015435195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:39:03.997538764Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:39:03.997773012Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:39:04.016115057Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T19:39:04.016129481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:39:04.016145852Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:39:04.016153542Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:39:04.016214578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:39:04.016248725Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-09T19:39:04.016258363Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:39:04.016277403Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.26µs success=true
level=debug ts=2026-05-09T19:39:04.016504543Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=debug ts=2026-05-09T19:39:04.078615917Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:39:07.262544637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:34220"
level=debug ts=2026-05-09T19:39:07.30998045Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T19:39:09.015515137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:39:10.223732754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:39:10.226947805Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:39:10.602063054Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=debug ts=2026-05-09T19:39:13.895209291Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=warn ts=2026-05-09T19:39:14.015201475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:39:17.188294533Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T19:39:18.983057648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:39:18.992177591Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:39:19.015865659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:39:19.21751672Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:39:20.480331114Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:39:20.480354324Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T19:39:23.514752395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="866 kB" avg_comp="866 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:39:23.519389745Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-09T19:39:23.773329342Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=warn ts=2026-05-09T19:39:24.016056884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:39:25.778345955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:39:27.065435798Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-09T19:39:27.379141765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:39:27.384930465Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T19:39:29.015262085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:39:29.817929572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:39:29.821168784Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:39:30.358645349Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-09T19:39:33.651129903Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T19:39:34.016011126Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:39:34.016034519Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:39:34.016042499Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:39:34.016042449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:39:36.943569166Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-09T19:39:37.265622933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:40302"
level=warn ts=2026-05-09T19:39:39.015727359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:39:40.236439212Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:39:40.236461113Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-09T19:39:40.660211666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:39:40.668965083Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:39:43.468432874Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:39:43.468445326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="164 kB" avg_uncomp="164 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:39:43.471890621Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T19:39:43.529148825Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-09T19:39:43.768780067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:39:43.771722621Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:39:44.015273388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:39:44.069489282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:39:44.072647237Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:39:44.369343269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="121 B" avg_comp="121 B" total_uncomp="252 kB" avg_uncomp="252 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:39:44.372575841Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:39:44.668870042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:39:44.67178511Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:39:45.52530174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:39:45.531516937Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:39:46.822157786Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-09T19:39:47.27437095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:39:47.2817663Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=warn ts=2026-05-09T19:39:49.015946847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:39:50.114857653Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=debug ts=2026-05-09T19:39:53.407378992Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-09T19:39:54.015986283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:39:54.205956792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:39:54.209196999Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T19:39:55.781562726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:39:56.189242325Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:39:56.699955504Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-09T19:39:56.780113627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:39:56.783524544Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:39:56.969137565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:39:56.971854646Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:39:57.148530643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:39:57.151149381Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T19:39:59.015956742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:39:59.483066535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:39:59.486064062Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:39:59.830512362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:39:59.83324223Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:39:59.993871844Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-09T19:39:59.993890495Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=6
level=debug ts=2026-05-09T19:40:03.28534363Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-09T19:40:03.997792579Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:40:03.998283101Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:40:04.015619118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:40:04.015627501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:40:04.015643233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:40:04.015653982Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:40:06.578456656Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T19:40:09.016209713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:40:09.912617748Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-09T19:40:09.912643595Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=3
level=info ts=2026-05-09T19:40:12.203875168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="866 kB" avg_comp="866 kB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-09T19:40:12.212845344Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e0e09a29e%3A19e0e3a88fc%3Ab0b0a70?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-09T19:40:12.212868391Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:40:12.212891368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:40:12.215543066Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:40:13.16303232Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-09T19:40:14.015710107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:40:14.89962214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:40:14.903275994Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:40:16.455906835Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T19:40:16.666476803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:40:16.674488787Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:40:17.452935172Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:40:17.452981932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:40:17.456226226Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:40:17.753995185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:40:17.757012523Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:40:18.053990715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:40:18.056863443Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T19:40:18.598422911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:40:18.601672299Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:40:19.016036829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:40:19.748979307Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T19:40:20.720369343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:40:20.726208831Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-09T19:40:21.078798892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48588"
level=debug ts=2026-05-09T19:40:23.04173111Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-09T19:40:24.0152018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:40:25.372900546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:40:25.377130907Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-09T19:40:25.784808692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:40:26.334339344Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-09T19:40:27.326124068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:40:27.332921008Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:40:29.015925872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:40:29.626452735Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:40:30.640500038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:40:30.64357638Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:40:31.772146811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:40:31.775036915Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:40:32.688755927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:40:32.691822921Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:40:32.859143779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:40:32.862111451Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:40:32.919405603Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-09T19:40:33.97406408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:40:33.976952793Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:40:34.015969883Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:40:34.015987125Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:40:34.01599445Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:40:34.016003948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:40:34.411205917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:40:34.413811152Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T19:40:35.136134143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:40:35.138939824Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:40:36.211901157Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
ts=2026-05-09T19:40:37.269475037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:35800"
level=info ts=2026-05-09T19:40:37.9048245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:40:37.913668546Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=info ts=2026-05-09T19:40:38.828145444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:40:38.83172131Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:40:39.015861009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:40:39.505170827Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-09T19:40:42.621566182Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:40:42.797250235Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=debug ts=2026-05-09T19:40:42.797275089Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-05-09T19:40:43.388783443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:40:43.396683898Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:40:44.015991192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:40:46.091491669Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T19:40:46.091519472Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-09T19:40:49.015956628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:40:49.38265533Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-09T19:40:51.435268152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:40:51.438211599Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:40:52.675820798Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-09T19:40:53.753885225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:40:53.755469797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-09T19:40:54.016049736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:40:55.787564781Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:40:55.968723649Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-09T19:40:56.096341905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:40:56.10165313Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T19:40:59.015856858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:40:59.26097022Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-09T19:41:02.288424574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="159 kB" avg_comp="159 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:41:02.291958552Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:41:02.486274014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:41:02.489231451Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T19:41:02.553401593Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-09T19:41:03.997329826Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:41:03.998392425Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:41:04.015728079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:41:04.015739349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:41:04.015754159Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:41:04.015766761Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:41:05.092397238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:41:05.095651728Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T19:41:05.846825356Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=info ts=2026-05-09T19:41:06.98165357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:41:06.987655734Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:41:06.987674784Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:41:06.987698197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:41:06.992908747Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T19:41:09.016203662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:41:09.138463599Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T19:41:09.931021493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:41:09.934145594Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:41:12.431515159Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T19:41:14.016078398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:41:14.204078363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:41:14.210183669Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-09T19:41:14.379536723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:50740"
level=debug ts=2026-05-09T19:41:15.725759933Z caller=checkpoint.go:372 msg="writing series" size="831 kB"
level=debug ts=2026-05-09T19:41:15.725782065Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=9
level=info ts=2026-05-09T19:41:16.027928936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:41:16.037038449Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:41:16.71816526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:41:16.722197243Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T19:41:18.075015538Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:41:18.666643541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:41:18.672394471Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T19:41:19.016201382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:41:19.017567598Z caller=checkpoint.go:372 msg="writing series" size="377 kB"
level=debug ts=2026-05-09T19:41:22.320479075Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T19:41:22.320500888Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=warn ts=2026-05-09T19:41:24.015629617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:41:24.814457118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:41:24.822524706Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:41:24.822544267Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:41:24.822565948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:41:24.829012155Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:41:25.431693076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:41:25.435150987Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:41:25.602769346Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
ts=2026-05-09T19:41:25.79035178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T19:41:26.079770892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=64d4d11b6f760481 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:41:26.088241611Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/64d4d11b6f760481/19e0e18e0b0%3A19e0e299221%3A56b43b9e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-kszn\"}"
level=error ts=2026-05-09T19:41:26.08827185Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:41:26.088292738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="32 kB" avg_uncomp="32 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:41:26.090859407Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:41:28.895593498Z caller=checkpoint.go:372 msg="writing series" size="482 kB"
level=warn ts=2026-05-09T19:41:29.01619416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:41:29.648224076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:41:29.652156715Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:41:31.732719803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:41:31.735581606Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:41:32.188053738Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=info ts=2026-05-09T19:41:34.015640949Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:41:34.015664565Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T19:41:34.015659682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:41:34.015672357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:41:34.266395489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:41:34.269829389Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:41:35.480674673Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T19:41:35.480697595Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=info ts=2026-05-09T19:41:37.321893288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:41:37.328328212Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:41:38.773807582Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
level=warn ts=2026-05-09T19:41:39.015644746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:41:42.066191367Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-09T19:41:42.343969465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:41:42.350498106Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T19:41:42.549988718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:41:42.556071215Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:41:42.83296736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:41:42.836296732Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:41:42.836317014Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:41:42.836337761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:41:42.838963652Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:41:44.001807361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:41:44.004783045Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T19:41:44.015950904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:41:44.635017945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:41:44.63877146Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:41:45.360363741Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T19:41:45.360383696Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=3
level=debug ts=2026-05-09T19:41:48.65081245Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T19:41:49.01577283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:41:49.022988435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:41:50.383810364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:41:50.387219979Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T19:41:51.94432667Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-05-09T19:41:53.109130685Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:41:53.450050827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:41:53.453053718Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T19:41:54.015727381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:41:55.236852458Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
ts=2026-05-09T19:41:55.792759201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T19:41:57.466754644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:41:57.470571637Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:41:58.531137805Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-09T19:41:58.531160648Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=4
level=info ts=2026-05-09T19:41:58.857770997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:41:58.860750491Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T19:41:59.015919337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:42:01.821799097Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-05-09T19:42:03.204375944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:42:03.209953451Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T19:42:03.997387276Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:42:03.998429428Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:42:04.015801568Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:42:04.015810788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:42:04.015816566Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:42:04.015824282Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:42:04.55220797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:42:04.559197698Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:42:04.559216743Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:42:04.559252365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:42:04.565201217Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:42:05.115687567Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-09T19:42:05.115711177Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=2
level=debug ts=2026-05-09T19:42:08.407765143Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-09T19:42:08.724926472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:42:08.728331789Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T19:42:08.832676062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:42:08.835499292Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T19:42:09.015699495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:42:09.267626009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:42:09.27466766Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:42:11.700818423Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=warn ts=2026-05-09T19:42:14.016108824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:42:14.734154305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:42:14.740752357Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:42:14.992385966Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=info ts=2026-05-09T19:42:18.21248398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:42:18.215610356Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:42:18.286185991Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-09T19:42:18.28620206Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-09T19:42:19.015846016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:42:21.57827898Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=debug ts=2026-05-09T19:42:22.475488215Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:42:22.57468467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:42:22.577963862Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T19:42:24.015871749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:42:24.87042547Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-09T19:42:25.794860011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T19:42:26.36685488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:42:26.370751095Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:42:28.163262249Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=info ts=2026-05-09T19:42:28.655490698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="99 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:42:28.663143027Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:42:28.804635357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:42:28.807461154Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-09T19:42:29.015610424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:42:30.932484796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:42:30.93874241Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:42:31.456221845Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:42:33.462132829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:42:33.466530366Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:42:34.016121775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:42:34.016148323Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:42:34.016155579Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:42:34.016192255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:42:34.748224615Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-09T19:42:36.049824896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T19:42:37.278217237Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47036"
level=info ts=2026-05-09T19:42:37.333877286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:42:37.336750129Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T19:42:38.041787292Z caller=checkpoint.go:372 msg="writing series" size="374 kB"
level=warn ts=2026-05-09T19:42:39.01572919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:42:39.311261723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:42:39.314423501Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T19:42:41.334088503Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T19:42:42.165327347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:42:42.16830059Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T19:42:44.015206871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:42:44.627250533Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-09T19:42:46.7928433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:42:46.796302534Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:42:47.919171352Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=debug ts=2026-05-09T19:42:47.957375039Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:42:49.015424028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:42:50.202514401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:42:50.209532971Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:42:51.212399025Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-09T19:42:51.212419341Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-09T19:42:52.971831602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:42:52.974952985Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T19:42:54.015347175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:42:54.519654529Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-09T19:42:54.519675105Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
ts=2026-05-09T19:42:55.797360929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:42:57.798323152Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T19:42:57.798345037Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-09T19:42:58.498690597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:42:58.501808372Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T19:42:59.015841346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:42:59.131141493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:42:59.13796343Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:42:59.778976026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:42:59.781962126Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:42:59.816929868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:42:59.819454748Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:43:01.089998855Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-09T19:43:03.997476513Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:43:03.998527051Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:43:04.015887086Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:43:04.015901659Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:43:04.01590878Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:43:04.015925719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:43:04.382841726Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-09T19:43:06.495922979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:43:06.500944226Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:43:07.675272403Z caller=checkpoint.go:372 msg="writing series" size="618 kB"
level=warn ts=2026-05-09T19:43:09.015977928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:43:09.799130414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:43:09.802921396Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:43:10.968345966Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-09T19:43:14.016122209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:43:14.261328097Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:43:14.261348567Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-09T19:43:17.553302282Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=debug ts=2026-05-09T19:43:18.066928196Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:43:19.016229221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:43:19.466994568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:43:19.469978659Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T19:43:19.6734768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:43:19.678613405Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:43:20.84688581Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-05-09T19:43:21.94599629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:43:22.151590448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:43:22.15415274Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T19:43:23.221205988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:43:23.224438279Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:43:23.398965866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:43:23.402262037Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:43:24.015639211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:43:24.138960687Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-09T19:43:24.522144437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:43:24.528402956Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-09T19:43:25.799507172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:43:27.432267323Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=warn ts=2026-05-09T19:43:29.015467359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:43:30.429284715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:43:30.433901838Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:43:30.724706106Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-05-09T19:43:34.015386866Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:43:34.015410123Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:43:34.015417757Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:43:34.015414498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:43:34.016552454Z caller=checkpoint.go:384 msg="flushing series" totalSize="831 kB" series=5
level=info ts=2026-05-09T19:43:34.022976793Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-09T19:43:34.024867366Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008453274s
ts=2026-05-09T19:43:37.282494196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:44938"
level=info ts=2026-05-09T19:43:38.320969999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:43:38.330007776Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:43:39.015747755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:43:39.886937134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:43:39.889878449Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T19:43:40.063382018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:43:40.066132623Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:43:40.353825369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:43:40.35682914Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:43:40.845197576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:43:40.848026601Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T19:43:44.015637775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:43:44.391853794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:51510"
level=warn ts=2026-05-09T19:43:49.015476487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:43:51.090574572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:43:51.099615859Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:43:52.162499546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:43:52.166332689Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:43:52.166352733Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:43:52.166376035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:43:52.169387713Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:43:53.981463525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:43:53.984572186Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T19:43:54.015551129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:43:54.521067582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T19:43:55.801428965Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=warn ts=2026-05-09T19:43:59.015683551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:44:03.997618364Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:44:03.997641051Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:44:04.016040474Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-09T19:44:04.016048173Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-09T19:44:04.016063233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:44:04.016073458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:44:04.016159651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:44:04.016348105Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-09T19:44:04.016358027Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:44:04.016382276Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.956µs success=true
level=debug ts=2026-05-09T19:44:04.016620152Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=info ts=2026-05-09T19:44:05.801326431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:44:05.806334786Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:44:07.309474833Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T19:44:09.015790946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:44:10.086862204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:44:10.102319701Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:44:10.213597793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:44:10.216571556Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T19:44:10.602617551Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:44:11.000740835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:44:11.003959385Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T19:44:13.019786927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:44:13.022799347Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:44:13.895265904Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=warn ts=2026-05-09T19:44:14.01573026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:44:15.79520689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:44:15.798453667Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T19:44:17.188176812Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-09T19:44:19.015218906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:44:19.092133543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:44:19.25587901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:44:19.263172784Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:44:19.301700156Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:44:20.480886545Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:44:20.480909293Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T19:44:20.608147117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:44:20.611475632Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:44:21.44182128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:44:21.445124475Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:44:23.382161052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:44:23.388503057Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:44:23.388522714Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:44:23.38854911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:44:23.394161635Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:44:23.773356018Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=warn ts=2026-05-09T19:44:24.016067478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:44:25.803225194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:44:27.066542353Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-09T19:44:29.016038904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:44:29.183495347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:44:29.187047113Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:44:30.358402525Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-09T19:44:33.651163111Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T19:44:34.015128057Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:44:34.015156991Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:44:34.015167608Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:44:34.015185715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:44:36.123360453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:44:36.131800226Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:44:36.943975014Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-09T19:44:38.28913676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:44:38.295870426Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:44:38.60977549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:44:38.612873357Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T19:44:39.015488902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:44:39.069644093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:44:39.072538373Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:44:40.237502393Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:44:40.237523898Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-09T19:44:41.027346814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:44:41.034358095Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:44:43.529705357Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-09T19:44:44.016106459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:44:44.396282989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:53668"
level=debug ts=2026-05-09T19:44:46.822229975Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-09T19:44:49.01617133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:44:49.552594415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:44:49.556341608Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:44:50.115133537Z caller=checkpoint.go:372 msg="writing series" size="611 kB"
ts=2026-05-09T19:44:51.105276611Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52410"
level=debug ts=2026-05-09T19:44:51.978139935Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:44:52.331301513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:44:52.334435678Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:44:53.407687346Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-09T19:44:53.512878365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="592 B" avg_comp="592 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:44:53.516204338Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:44:54.015906929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:44:55.804302463Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:44:56.700390725Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-09T19:44:58.347300765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:44:58.353750315Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:44:59.015502117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:44:59.993305193Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-09T19:44:59.993325974Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=6
level=info ts=2026-05-09T19:45:00.965609663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:45:00.971291478Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:45:02.034418249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:45:02.037312283Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:45:02.866617652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:45:02.871794323Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:45:03.284970718Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-09T19:45:03.974971518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:45:03.978043401Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:45:03.997483105Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:45:03.998529127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:45:03.999781114Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T19:45:03.999819388Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975950 n=1
level=debug ts=2026-05-09T19:45:03.999860525Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975950 n=1
level=info ts=2026-05-09T19:45:04.015876805Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:45:04.015880305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:45:04.015894404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:45:04.015901934Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:45:06.578011385Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-09T19:45:09.015710998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:45:09.418706599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:45:09.422213651Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:45:09.645217364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:45:09.648393647Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T19:45:09.919279642Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-09T19:45:09.919309468Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=3
level=debug ts=2026-05-09T19:45:13.163283154Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-09T19:45:14.016048419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:45:14.399211455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:60580"
level=debug ts=2026-05-09T19:45:15.051002772Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:45:15.052626794Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T19:45:15.835751183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:45:15.838806409Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T19:45:16.456681856Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T19:45:18.83225665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:45:18.835859906Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:45:19.015822616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:45:19.748896922Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-09T19:45:21.10615717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57132"
level=info ts=2026-05-09T19:45:22.896627672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:45:22.903653128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:45:23.042030711Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=warn ts=2026-05-09T19:45:24.015711198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:45:25.811229316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:45:26.290520088Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:45:26.33484651Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-09T19:45:29.015490002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:45:29.314779738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:45:29.322750204Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:45:29.626869088Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:45:30.546405927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:45:30.569971096Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:45:30.639442154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:45:30.64307296Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:45:32.919794067Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=info ts=2026-05-09T19:45:33.045229834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:45:33.049658421Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:45:34.015265127Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:45:34.015294407Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:45:34.015303318Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:45:34.015348581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:45:35.666451422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:45:35.669834096Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:45:36.212737744Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-09T19:45:39.016021667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:45:39.504724582Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-09T19:45:42.252174051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:45:42.257296356Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:45:42.257324972Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:45:42.257354661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:45:42.260053715Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T19:45:42.498837437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:45:42.502995974Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:45:42.79811214Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=debug ts=2026-05-09T19:45:42.79813348Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-05-09T19:45:44.015684434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:45:44.402702074Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:40714"
level=info ts=2026-05-09T19:45:45.500594801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:45:45.509052566Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:45:46.091669617Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T19:45:46.091696611Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-09T19:45:49.015472332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:45:49.382985455Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-09T19:45:50.860316561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:45:50.866252013Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:45:52.251339255Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:45:52.675924211Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T19:45:54.015797425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:45:55.813445723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:45:55.968357836Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-09T19:45:56.148009144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:45:56.151312757Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:45:56.336985176Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:45:56.457267172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:45:56.460361138Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-09T19:45:56.460378307Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:45:56.460397716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:45:56.462809403Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:45:57.81982904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:45:57.822906162Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:45:59.015995621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:45:59.261069596Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-09T19:46:02.378947085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:46:02.382362128Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T19:46:02.553735658Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T19:46:03.582651718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:46:03.59436311Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:46:03.9975152Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:46:03.99826064Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=warn ts=2026-05-09T19:46:04.015613126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:46:04.015616802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:46:04.015638148Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:46:04.015647102Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:46:04.896752744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:46:04.899680636Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:46:05.846759491Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=info ts=2026-05-09T19:46:06.892862227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:46:06.89633019Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T19:46:07.29797507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:39752"
level=info ts=2026-05-09T19:46:07.886763517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:46:07.889938621Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:46:09.016153761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:46:09.1394987Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T19:46:11.983209051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:46:11.986420036Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:46:12.432161146Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-09T19:46:12.767009243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:46:12.775810901Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:46:14.015224639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:46:15.72501725Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-05-09T19:46:15.725040964Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=9
level=info ts=2026-05-09T19:46:17.941348357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:46:17.945211006Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:46:19.016206609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:46:19.017360221Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
ts=2026-05-09T19:46:21.110927489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59932"
level=debug ts=2026-05-09T19:46:22.321418033Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T19:46:22.321439817Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=info ts=2026-05-09T19:46:23.455645898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:46:23.459317561Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:46:23.496156446Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:46:24.0159227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:46:25.470044947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:46:25.473844938Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:46:25.602560032Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
ts=2026-05-09T19:46:25.816518298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:46:26.620438833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:46:26.627081081Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:46:27.782046079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:46:27.785277365Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:46:27.785294794Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:46:27.785320421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:46:27.788123935Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T19:46:28.895749065Z caller=checkpoint.go:372 msg="writing series" size="488 kB"
level=warn ts=2026-05-09T19:46:29.015721705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:46:29.317724601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:46:29.320544332Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:46:29.320564583Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:46:29.320584576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:46:29.322833737Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:46:32.188330276Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-05-09T19:46:34.015762974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:46:34.015789365Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:46:34.015797885Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:46:34.015805447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:46:34.539971024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:46:34.544217706Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:46:34.544238937Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:46:34.544266475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:46:34.546776578Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:46:34.634039743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:46:34.637076718Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:46:35.481489401Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T19:46:35.481510063Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=info ts=2026-05-09T19:46:38.268603839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:46:38.2723857Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-09T19:46:38.272406794Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:46:38.27242745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:46:38.274799789Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:46:38.773609249Z caller=checkpoint.go:372 msg="writing series" size="538 kB"
level=warn ts=2026-05-09T19:46:39.015708752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:46:40.121905649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:46:40.12898049Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T19:46:40.498752154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:46:40.501930399Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:46:40.688430259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:46:40.691484677Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T19:46:42.0664342Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-09T19:46:44.016032169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:46:45.360649427Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T19:46:45.360674553Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=3
level=info ts=2026-05-09T19:46:45.592029174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:46:45.5952347Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:46:46.637104988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:46:46.644210224Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:46:46.979546225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:46:48.581116195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:46:48.584501305Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:46:48.650964495Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T19:46:49.015829026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:46:51.575677793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:46:51.579048161Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:46:51.579068965Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:46:51.579088065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:46:51.581421879Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T19:46:51.9444532Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-09T19:46:54.015546668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:46:55.236922979Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
ts=2026-05-09T19:46:55.819924307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:46:56.363989178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:46:56.368388691Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:46:56.368409168Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:46:56.368433404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:46:56.371530257Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:46:56.553048217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:46:56.556105333Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T19:46:57.221511907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:46:57.224849834Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:46:57.22486677Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:46:57.224885899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:46:57.227376195Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:46:57.5382735Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:46:58.531146913Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-09T19:46:58.531168273Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=4
level=warn ts=2026-05-09T19:46:59.015216992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:47:01.822385828Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=debug ts=2026-05-09T19:47:03.997458049Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:47:03.997525845Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:47:04.015957804Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:47:04.015975102Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:47:04.015983233Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:47:04.015982274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:47:04.402255831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:47:04.408359728Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:47:05.117043799Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-09T19:47:05.117066365Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=info ts=2026-05-09T19:47:07.337907281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:47:07.342262236Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T19:47:08.305968677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:47:08.309414439Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T19:47:08.407835882Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-09T19:47:09.015491718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:47:09.190989726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:47:09.197796309Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:47:11.699919628Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=info ts=2026-05-09T19:47:11.707674705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:47:11.71107158Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:47:13.998630584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:47:14.001757297Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T19:47:14.015924267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:47:14.410923177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:54410"
level=info ts=2026-05-09T19:47:14.940089884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:47:14.942801205Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:47:14.993048542Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=info ts=2026-05-09T19:47:15.361253815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:47:15.364173834Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:47:16.662970654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:47:16.665930533Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T19:47:17.938469373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:47:17.941246892Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:47:18.285793777Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-09T19:47:18.285817828Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-09T19:47:19.015568069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:47:19.714566717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:47:19.717353255Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:47:21.578052723Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-09T19:47:23.522670017Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:47:23.771424035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:47:23.774972802Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:47:23.943671038Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:47:24.0158716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:47:24.870426424Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-09T19:47:25.821919566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:47:28.163308881Z caller=checkpoint.go:372 msg="writing series" size="379 kB"
level=info ts=2026-05-09T19:47:28.622653323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:47:28.627698718Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T19:47:29.015390295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:47:30.886291752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:47:30.892018061Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:47:30.89203802Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:47:30.892061223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:47:30.897230732Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:47:31.455878212Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:47:34.015681211Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:47:34.015707787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:47:34.015716153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:47:34.015769973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:47:34.749187064Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-09T19:47:37.308243437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:46006"
level=info ts=2026-05-09T19:47:37.74459806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:47:37.748256869Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:47:38.041424742Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-09T19:47:38.516670163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:47:38.519919199Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:47:38.773441362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:47:38.780839333Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:47:39.016202452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:47:41.334192774Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T19:47:41.982917479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:47:41.986606531Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:47:44.015601919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:47:44.222080193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:47:44.225204073Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T19:47:44.41362383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:51434"
level=debug ts=2026-05-09T19:47:44.626413824Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-09T19:47:45.713349861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:47:45.719626018Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:47:47.919684365Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T19:47:49.015804154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:47:51.212289905Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-09T19:47:54.015262517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:47:54.522660713Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-09T19:47:54.522685896Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=11
level=info ts=2026-05-09T19:47:55.298706659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:47:55.30216526Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:47:55.47250869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:47:55.476085521Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-09T19:47:55.824490959Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T19:47:57.494693547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:47:57.501647024Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:47:57.798266609Z caller=checkpoint.go:372 msg="writing series" size="968 kB"
level=debug ts=2026-05-09T19:47:58.158622989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:47:58.551839915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:47:58.554925101Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T19:47:59.015559912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:48:01.090128864Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-09T19:48:02.139404474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:48:02.145386063Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:48:02.145410879Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:48:02.145435488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:48:02.150889871Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:48:02.807495794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:48:03.997779523Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:48:03.998160141Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:48:04.015514142Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:48:04.015521738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:48:04.015531735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:48:04.015539674Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:48:04.382874638Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-09T19:48:04.382898657Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-09T19:48:06.725971509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:48:06.73035483Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T19:48:07.311831933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:36548"
level=debug ts=2026-05-09T19:48:07.676554038Z caller=checkpoint.go:372 msg="writing series" size="618 kB"
level=warn ts=2026-05-09T19:48:09.015651091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:48:10.968850948Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-09T19:48:12.790694083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:48:12.793776764Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:48:14.015215072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:48:14.261796417Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:48:14.261816148Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
ts=2026-05-09T19:48:14.41624369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:43502"
level=info ts=2026-05-09T19:48:15.902034663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:48:15.905831602Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:48:17.553595197Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T19:48:18.035292289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:48:18.038902411Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:48:18.038921874Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:48:18.038942344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:48:18.041619962Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T19:48:18.778561482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:48:18.784982995Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:48:19.015534926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:48:19.505811906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:48:19.512947288Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:48:19.512969174Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:48:19.513007136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:48:19.519483313Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:48:19.62048164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:48:19.623433308Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:48:20.737555567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:48:20.74070533Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:48:20.846200014Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-09T19:48:22.140875076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:48:22.146405124Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T19:48:24.015747047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:48:24.139000204Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
ts=2026-05-09T19:48:25.827950537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:48:27.01205588Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:48:27.43241534Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=info ts=2026-05-09T19:48:28.213542614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:48:28.217009466Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T19:48:29.01565481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:48:30.7249727Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-09T19:48:34.016041201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:48:34.01604787Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:48:34.016072937Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:48:34.016081232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:48:34.017122141Z caller=checkpoint.go:384 msg="flushing series" totalSize="838 kB" series=5
level=info ts=2026-05-09T19:48:34.024682709Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-09T19:48:34.025980868Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00946391s
level=info ts=2026-05-09T19:48:34.937647697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:48:34.941405147Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T19:48:35.526010357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:48:35.529378162Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:48:37.087036566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:48:37.090465559Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:48:37.090482931Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:48:37.090504076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:48:37.093121347Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:48:38.710755753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:48:38.718367545Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:48:38.837732022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:48:38.84038728Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T19:48:39.015782139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:48:43.130993722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:48:43.138873174Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:48:44.015196967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:48:46.423966281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:48:46.429824983Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:48:46.946011348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:48:46.9499401Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:48:49.016135514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:48:50.644970804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:48:50.64657659Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T19:48:51.764991083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:48:51.767914537Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:48:52.075863929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:48:52.082638434Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:48:52.282206202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:48:52.285386534Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:48:54.015524803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:48:54.239858237Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T19:48:55.831667989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:48:56.618629803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:48:56.621961932Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T19:48:56.855573775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:48:56.858837562Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-09T19:48:59.015813606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:49:02.945773719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:49:02.953565074Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:49:03.997763323Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:49:03.998189902Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:49:04.015531742Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:49:04.015553102Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T19:49:04.015537731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:49:04.015561412Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:49:04.015560231Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T19:49:04.015708291Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-09T19:49:04.015717401Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:49:04.015734939Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.935µs success=true
level=debug ts=2026-05-09T19:49:04.016007209Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-09T19:49:05.176284854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:49:05.181813993Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:49:07.309028779Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-09T19:49:08.87538495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:49:08.878481742Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T19:49:09.015955909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:49:10.60184915Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-09T19:49:13.894957616Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=warn ts=2026-05-09T19:49:14.016054406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:49:17.187679258Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T19:49:17.368222753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:49:17.37573351Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:49:18.943087258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:49:19.016200595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:49:19.456520567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:49:19.460254359Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:49:20.38187853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:49:20.385244964Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:49:20.47995697Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:49:20.479977363Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-09T19:49:23.772644222Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
level=warn ts=2026-05-09T19:49:24.015339006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:49:25.835567125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:49:26.467942128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:49:26.471078759Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:49:27.065002788Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=info ts=2026-05-09T19:49:27.892292637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:49:27.89536808Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T19:49:29.015552119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:49:30.357706904Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T19:49:31.433848746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:49:31.437323668Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:49:32.893692771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:49:32.897864152Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:49:33.651119755Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T19:49:34.015626976Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:49:34.01564189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:49:34.015654206Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:49:34.015662129Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T19:49:34.619356075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:49:34.622222131Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T19:49:36.131780702Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:49:36.823082891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="592 B" avg_comp="592 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:49:36.826185701Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T19:49:36.943434518Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-09T19:49:37.319788103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:45306"
level=info ts=2026-05-09T19:49:38.629073246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:49:38.632527559Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-09T19:49:39.015479046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:49:40.236421518Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:49:40.236442312Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-05-09T19:49:40.783601952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:49:40.786505869Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:49:41.429352443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:49:41.435635889Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:49:43.529237513Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-09T19:49:44.015530864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:49:44.424283945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:57608"
level=debug ts=2026-05-09T19:49:46.821418872Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-09T19:49:49.007273882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:49:49.015539226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-09T19:49:49.016642278Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:49:50.114761915Z caller=checkpoint.go:372 msg="writing series" size="482 kB"
level=info ts=2026-05-09T19:49:50.56939017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:49:50.572463273Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:49:51.352619215Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:49:53.406884862Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-09T19:49:54.015618261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:49:54.698554208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:49:54.70584456Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T19:49:55.838740671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:49:56.698928495Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-09T19:49:57.95439556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:49:57.959931285Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T19:49:59.016165234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:49:59.99354311Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-09T19:49:59.993563801Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=6
level=info ts=2026-05-09T19:50:01.096335307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:50:01.099745725Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:50:03.285141368Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-09T19:50:03.99767573Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:50:03.997709689Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:50:04.016026137Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:50:04.016055367Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:50:04.016065715Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:50:04.016128903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:50:06.57781352Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-09T19:50:06.679161236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:50:06.683736281Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-09T19:50:07.32348271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:55606"
level=warn ts=2026-05-09T19:50:09.016169439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:50:09.918872889Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-09T19:50:09.918898671Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=3
level=debug ts=2026-05-09T19:50:13.162678512Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-09T19:50:13.19619623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:50:14.016024195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:50:14.427335291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:58272"
level=debug ts=2026-05-09T19:50:16.45613221Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T19:50:18.097402691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:50:18.102453285Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:50:19.015752071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:50:19.748014662Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-09T19:50:21.134935777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50486"
level=debug ts=2026-05-09T19:50:21.195972578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:50:21.747426053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:50:21.754499568Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:50:21.971049521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:50:21.97459112Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:50:21.974613987Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:50:21.974636688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:50:21.977606875Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:50:23.002955312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:50:23.005844659Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:50:23.041230862Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=warn ts=2026-05-09T19:50:24.015901969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:50:25.844590884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:50:26.334166342Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-05-09T19:50:26.699978015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:50:26.703241305Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:50:26.79461409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:50:26.797655563Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T19:50:26.838741992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:50:26.84613368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:50:26.846154212Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:50:26.846183487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:50:26.852956289Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:50:27.373286588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:50:27.376697814Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:50:29.016112789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:50:29.626780884Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:50:29.660894962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:50:29.66423143Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T19:50:32.80602261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:50:32.80969925Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:50:32.919195603Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=info ts=2026-05-09T19:50:34.016025645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:50:34.016053247Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:50:34.016061093Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:50:34.016081505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:50:35.975515959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:50:35.979186382Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:50:36.212157401Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-05-09T19:50:37.410349255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:50:37.417849971Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:50:38.170873908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:50:38.173960438Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T19:50:39.01620875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:50:39.504501211Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-09T19:50:39.5045239Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-05-09T19:50:41.40585768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:50:41.408756199Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:50:41.872236033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:50:41.87966528Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:50:42.797286456Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=warn ts=2026-05-09T19:50:44.015694707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:50:46.091567481Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T19:50:46.09160245Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-05-09T19:50:46.812875853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:50:46.819808816Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:50:49.015656658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:50:49.382862011Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-09T19:50:52.675355715Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T19:50:54.015425677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:50:55.754550092Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T19:50:55.848138373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:50:55.967961797Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-09T19:50:56.042498162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:50:56.04972371Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T19:50:57.587733209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:50:57.591236866Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T19:50:59.015822108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:50:59.260046213Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-09T19:50:59.357217532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:50:59.361308324Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:51:02.553119223Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T19:51:03.99759228Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:51:03.997607432Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:51:04.016060355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:51:04.016079071Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:51:04.016086075Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:51:04.016115192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:51:05.13955242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:51:05.143027621Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:51:05.846289306Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-09T19:51:05.849239925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:51:05.856181121Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:51:08.837977592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:51:09.015821339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:51:09.138143481Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T19:51:10.430241261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:51:10.433455182Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:51:12.430934664Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T19:51:14.015250114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:51:15.724237997Z caller=checkpoint.go:372 msg="writing series" size="851 kB"
level=debug ts=2026-05-09T19:51:15.724260292Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=9
level=info ts=2026-05-09T19:51:16.754219753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:51:16.757436569Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T19:51:16.821770444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:51:16.824483387Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:51:18.448901617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:51:18.452107412Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:51:18.567318983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:51:18.570557422Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:51:18.929761285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:51:18.932764961Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:51:19.015913455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:51:19.016132145Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
ts=2026-05-09T19:51:21.139691923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35038"
level=debug ts=2026-05-09T19:51:21.277758237Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:51:21.982369314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:51:21.985601237Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:51:22.319793154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T19:51:22.320049755Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T19:51:22.320066925Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=error ts=2026-05-09T19:51:22.323364797Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T19:51:23.993267703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:51:24.000537994Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:51:24.015716583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:51:25.60237702Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
ts=2026-05-09T19:51:25.851189558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:51:28.895486796Z caller=checkpoint.go:372 msg="writing series" size="511 kB"
level=warn ts=2026-05-09T19:51:29.016138142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:51:32.187640959Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-09T19:51:34.016013058Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:51:34.016046939Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:51:34.016054652Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:51:34.016043602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:51:34.796071771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:51:34.801426526Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:51:35.480985917Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T19:51:35.481007973Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=4
level=debug ts=2026-05-09T19:51:38.772406385Z caller=checkpoint.go:372 msg="writing series" size="427 kB"
level=warn ts=2026-05-09T19:51:39.015870691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:51:41.573626324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:51:41.578766588Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:51:42.065350382Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-09T19:51:44.015999123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:51:44.246134878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:51:44.253625979Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:51:45.359593771Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T19:51:45.359614907Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=3
level=info ts=2026-05-09T19:51:46.329939776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:51:46.336444594Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:51:46.427910846Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:51:48.651114539Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T19:51:49.015839189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:51:50.239508069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:51:50.243778954Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:51:51.943433707Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-09T19:51:53.599437498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:51:53.602801364Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:51:54.016071792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:51:54.691397899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:51:54.698142915Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:51:55.236147304Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
ts=2026-05-09T19:51:55.853064023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:51:58.529956907Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-09T19:51:58.529982796Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=4
level=warn ts=2026-05-09T19:51:59.01616504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:52:00.574005738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:52:00.578497908Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:52:00.765011295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:52:00.768165656Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:52:01.821191337Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=debug ts=2026-05-09T19:52:03.648892878Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:52:03.997512928Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:52:03.997870438Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:52:04.015126264Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:52:04.015148501Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:52:04.015156925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:52:04.015161787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:52:05.115963947Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-09T19:52:05.115985894Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
level=info ts=2026-05-09T19:52:05.375313917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:52:05.379435502Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:52:05.379457066Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:52:05.379479274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:52:05.381805032Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T19:52:07.05776775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:52:07.060878654Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T19:52:08.406882309Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-09T19:52:09.015216595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:52:09.238350156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:52:09.245701537Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:52:09.653389517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:52:09.656505311Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T19:52:11.69962846Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=info ts=2026-05-09T19:52:11.831976487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:52:11.838083268Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T19:52:14.015845958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:52:14.976216052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:52:14.979614163Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:52:14.991900968Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-09T19:52:18.116314448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=debug ts=2026-05-09T19:52:18.118621982Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-09T19:52:18.119413105Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-09T19:52:18.119429151Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:52:18.119448959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:52:18.122020026Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:52:18.28482013Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-09T19:52:18.284841366Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-05-09T19:52:19.015323362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:52:21.577146495Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-09T19:52:23.838214534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:52:23.84197578Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:52:24.015282009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:52:24.870352724Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-09T19:52:25.468751847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:52:25.472352876Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T19:52:25.854836885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:52:28.163614577Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=info ts=2026-05-09T19:52:28.533279394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:52:28.536626597Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T19:52:29.015375486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:52:31.455700911Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:52:31.622214675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:52:31.631418199Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:52:33.150470215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:52:33.153543176Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:52:34.015863064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:52:34.015886893Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T19:52:34.015890828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:52:34.015903996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:52:34.748654217Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-09T19:52:35.900864241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:52:35.90997789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:52:38.040925879Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=warn ts=2026-05-09T19:52:39.015650567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:52:39.376668074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:52:39.378208275Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T19:52:40.562979038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:52:40.568145432Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T19:52:41.160903732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:52:41.163906985Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:52:41.333496812Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T19:52:42.674145846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:52:42.67925262Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:52:42.67927229Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:52:42.679297755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:52:42.681571194Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:52:43.35958782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:52:43.36252348Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:52:43.362543965Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:52:43.362564503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:52:43.364834554Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:52:43.90056236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:52:43.903349227Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:52:44.015182335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:52:44.44261538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:60956"
level=debug ts=2026-05-09T19:52:44.626571883Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-09T19:52:47.919248287Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=info ts=2026-05-09T19:52:48.477753571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:52:48.481402689Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:52:49.015679798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:52:51.211584147Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-09T19:52:51.211605544Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=debug ts=2026-05-09T19:52:51.893476266Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:52:52.94441093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:52:52.947739459Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T19:52:54.0160696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:52:54.524061235Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-09T19:52:54.524081859Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=1
level=info ts=2026-05-09T19:52:54.56768339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:52:54.570916152Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:52:54.570935694Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:52:54.57095899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:52:54.573506748Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-09T19:52:55.857420413Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:52:56.37756984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:52:56.380661168Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:52:57.797706034Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T19:52:57.79772862Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=1
level=info ts=2026-05-09T19:52:57.844800169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:52:57.848070364Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:52:57.927381729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:52:57.935343478Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:52:59.015391654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:52:59.869920875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:52:59.872895147Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-09T19:52:59.872912614Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:52:59.872934765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:52:59.875246679Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:53:01.08944178Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-09T19:53:03.997544496Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:53:03.998292044Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:53:04.015632648Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:53:04.015652208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:53:04.015663635Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:53:04.015683514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:53:04.382753855Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
ts=2026-05-09T19:53:07.337079007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:56454"
level=info ts=2026-05-09T19:53:07.341404755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:53:07.347587706Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:53:07.675867884Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=info ts=2026-05-09T19:53:08.72171648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:53:08.725262758Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T19:53:09.015993427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:53:10.509230342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:53:10.512467713Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:53:10.915557185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:53:10.918895719Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:53:10.918918846Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:53:10.918937796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:53:10.92133296Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:53:10.967630352Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-09T19:53:14.01595314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:53:14.260302159Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:53:14.260329488Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
ts=2026-05-09T19:53:14.44563673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:35210"
level=info ts=2026-05-09T19:53:15.38100009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:53:15.386061431Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T19:53:17.109368658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:53:17.112753174Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:53:17.165814707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:53:17.16861048Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:53:17.552708008Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-09T19:53:19.01588709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:53:20.097247307Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:53:20.845362352Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T19:53:22.148152176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:53:22.153666245Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:53:22.997212022Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:53:23.291728937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:53:23.294616027Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:53:23.526218817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:53:23.532234846Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:53:24.015487413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:53:24.138903181Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-09T19:53:24.638780076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:53:24.641764431Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:53:25.125031269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:53:25.127802507Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T19:53:25.861211228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:53:27.431663372Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=warn ts=2026-05-09T19:53:29.015770442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:53:30.724014064Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-09T19:53:30.884430582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:53:30.891806199Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:53:34.015734236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:53:34.015741763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:53:34.015758921Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:53:34.015766563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:53:34.016843743Z caller=checkpoint.go:384 msg="flushing series" totalSize="842 kB" series=5
level=info ts=2026-05-09T19:53:34.023505298Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-09T19:53:34.025028351Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009117448s
level=info ts=2026-05-09T19:53:34.42720834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:53:34.43025465Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-09T19:53:37.339958684Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:54984"
level=info ts=2026-05-09T19:53:38.091780364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="455 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:53:38.100157828Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:53:39.015510615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-09T19:53:44.01557857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:53:44.355143371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:53:44.358682451Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T19:53:44.500889673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:53:44.504227962Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:53:44.504250732Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:53:44.504276266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:53:44.506758708Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:53:45.273709995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:53:45.27759829Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:53:48.308652263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:53:48.313206923Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T19:53:49.016003629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:53:50.969223561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:53:50.972323515Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:53:50.972345526Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:53:50.972367189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:53:50.974860725Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T19:53:51.153681679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50952"
level=debug ts=2026-05-09T19:53:51.19751598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:53:54.016067035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:53:54.669396095Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:53:54.956393671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:53:54.959199107Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T19:53:55.86464551Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T19:53:55.99373087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:53:55.997184675Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:53:56.040343522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:53:56.043219367Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T19:53:59.015983018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:54:03.997363419Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:54:03.998494526Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:54:04.015840895Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T19:54:04.015851068Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:54:04.015868868Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:54:04.015877206Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:54:04.015931368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:54:04.015972647Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-09T19:54:04.015980602Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:54:04.015997132Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.906µs success=true
level=debug ts=2026-05-09T19:54:04.016265043Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-09T19:54:04.140513341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:54:04.145822248Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:54:04.145842945Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:54:04.145869195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:54:04.150896677Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T19:54:05.623894175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:54:05.627119706Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T19:54:06.936559657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:54:06.94002728Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:54:07.309273276Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-09T19:54:09.015962349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:54:10.254446864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:54:10.262011629Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T19:54:10.602086872Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:54:10.83575085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:54:10.838930053Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:54:10.845971408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:54:10.852223162Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:54:11.44299609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:54:11.450304737Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:54:13.895377057Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=warn ts=2026-05-09T19:54:14.015527414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:54:17.187873726Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T19:54:17.194049936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:54:17.197160452Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:54:17.368399691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:54:17.374653Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T19:54:18.271336693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:54:18.277728946Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:54:18.277747317Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:54:18.277779904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:54:18.283713601Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:54:18.304014434Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T19:54:19.015689979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:54:20.480649274Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:54:20.480675234Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T19:54:20.484802725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="592 B" avg_comp="592 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:54:20.488944601Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T19:54:23.773417844Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-09T19:54:24.015710008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:54:25.86812021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:54:26.214902266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:54:26.218366423Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T19:54:27.066044609Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-09T19:54:27.132200053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:54:27.136035629Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:54:29.016147753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:54:30.358331686Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T19:54:30.770371017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:54:30.777517805Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:54:33.65062131Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T19:54:34.015273839Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:54:34.015298855Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:54:34.015305947Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:54:34.015307022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:54:34.432025178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:54:34.434961814Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:54:35.372943458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:54:35.375968858Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:54:35.663466072Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:54:36.795730563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:54:36.798661438Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T19:54:36.943982454Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T19:54:39.015912581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:54:40.23654669Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:54:40.236568985Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-05-09T19:54:43.529240048Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-09T19:54:44.016026561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:54:44.45273719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:51718"
level=info ts=2026-05-09T19:54:46.2508481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:54:46.255422825Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:54:46.822059857Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-09T19:54:49.015973187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:54:49.375626019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:54:49.379028755Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:54:50.115176244Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=info ts=2026-05-09T19:54:52.389118093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:54:52.395552412Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:54:53.407534089Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-09T19:54:54.015196105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:54:54.060472724Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T19:54:55.871469509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T19:54:56.699428041Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-09T19:54:58.688901629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:54:58.692724885Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T19:54:59.015466336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:54:59.994882871Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-09T19:54:59.994902882Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=info ts=2026-05-09T19:55:01.044528559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:55:01.047774079Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T19:55:01.236294438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:55:01.240138816Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:55:01.240157289Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:55:01.240177996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:55:01.243294381Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:55:01.696749954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:55:01.703813513Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:55:02.966336027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:55:02.96975487Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:55:02.969772227Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:55:02.969792588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:55:02.972375486Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:55:03.285496319Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-09T19:55:03.997571617Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:55:03.997805092Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:55:04.015128524Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:55:04.015148769Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:55:04.015156258Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:55:04.015176717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:55:05.4067543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:55:05.414528082Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:55:05.705310388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:55:05.712230919Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:55:05.712250382Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:55:05.712274624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:55:05.719500106Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T19:55:06.577972744Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
ts=2026-05-09T19:55:07.348813921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:55294"
level=info ts=2026-05-09T19:55:07.773495287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:55:07.777032247Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:55:08.742850404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:55:08.745805588Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T19:55:09.01612692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:55:09.921978875Z caller=checkpoint.go:372 msg="writing series" size="57 MB"
level=debug ts=2026-05-09T19:55:09.922005245Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=3
level=info ts=2026-05-09T19:55:12.415998077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:55:12.419567088Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:55:13.163544191Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-09T19:55:14.016238778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:55:14.945722993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:55:14.948810816Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:55:16.455581683Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T19:55:16.998212614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:55:17.001255527Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T19:55:19.015744987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:55:19.423665149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T19:55:19.748668641Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T19:55:20.71897711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:55:20.726210432Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:55:20.738254075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:55:20.741643042Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:55:23.041748156Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=warn ts=2026-05-09T19:55:24.015606092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:55:24.851502189Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:55:24.999586361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:55:25.003949095Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-09T19:55:25.873871121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:55:26.334207535Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-09T19:55:27.282555713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:55:27.285466049Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T19:55:29.015550269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:55:29.626046409Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-09T19:55:31.825657876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:55:31.828928839Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:55:32.561992031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:55:32.565348423Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:55:32.919688156Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-09T19:55:34.015808374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:55:34.015832905Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:55:34.01584255Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:55:34.015858433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:55:36.211578071Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=warn ts=2026-05-09T19:55:39.015876463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:55:39.504745564Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=debug ts=2026-05-09T19:55:39.504768497Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-05-09T19:55:41.377927822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:55:41.381666148Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T19:55:41.858145146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:55:41.861451644Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:55:42.797246244Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=info ts=2026-05-09T19:55:43.292667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:55:43.299778801Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:55:44.015471773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:55:44.457864841Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:47614"
level=info ts=2026-05-09T19:55:45.505817168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:55:45.514711583Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:55:46.090784693Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T19:55:46.090808158Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=warn ts=2026-05-09T19:55:49.016106004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:55:49.382752569Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
ts=2026-05-09T19:55:51.164455624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57018"
level=info ts=2026-05-09T19:55:51.465721352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:55:51.472811973Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:55:51.935067728Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:55:52.67536641Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T19:55:53.481751899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:55:53.485446236Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:55:54.01523263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:55:55.598888744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:55:55.601901418Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-09T19:55:55.875930223Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T19:55:55.967838048Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=debug ts=2026-05-09T19:55:57.269984743Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T19:55:57.271458586Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T19:55:58.392828845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:55:58.395663874Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T19:55:59.015401649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:55:59.260574558Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-09T19:56:02.155537697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:56:02.163402161Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:56:02.553586601Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T19:56:03.366903234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:56:03.369923768Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:56:03.397240117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:56:03.399880785Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:56:03.99741694Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:56:03.998466717Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:56:04.015810975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:56:04.01582782Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:56:04.015834941Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:56:04.015836747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:56:05.84648043Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=warn ts=2026-05-09T19:56:09.016205968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:56:09.138538599Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T19:56:09.608758826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:56:09.611923999Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T19:56:12.43188827Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T19:56:14.015427694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:56:15.724709424Z caller=checkpoint.go:372 msg="writing series" size="862 kB"
level=debug ts=2026-05-09T19:56:15.724730718Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=9
level=info ts=2026-05-09T19:56:15.95143045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:56:15.954961663Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T19:56:19.015208033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:56:19.016452163Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=info ts=2026-05-09T19:56:20.503964111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:56:20.507285178Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:56:22.321960725Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T19:56:22.321984115Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=info ts=2026-05-09T19:56:23.04764238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:56:23.054721183Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:56:23.761877093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:56:23.765316764Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T19:56:24.015184064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:56:25.602224344Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-09T19:56:25.779579333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T19:56:25.878147336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T19:56:28.542565206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:56:28.549232894Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:56:28.895641804Z caller=checkpoint.go:372 msg="writing series" size="516 kB"
level=warn ts=2026-05-09T19:56:29.015284322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:56:31.070408304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:56:31.073437028Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T19:56:32.187895563Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-09T19:56:32.349203523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:56:32.355972978Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:56:34.015497342Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T19:56:34.015504028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:56:34.015525843Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:56:34.015535619Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T19:56:35.481230059Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T19:56:35.481253646Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=4
level=debug ts=2026-05-09T19:56:38.773874083Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=warn ts=2026-05-09T19:56:39.01571108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:56:42.065752329Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-09T19:56:44.016158767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:56:44.720145963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:56:44.726324789Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:56:45.055262781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:56:45.058458502Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:56:45.177725216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:56:45.184475589Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:56:45.184496161Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:56:45.184526442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:56:45.19023187Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:56:45.36039113Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T19:56:45.360412052Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=3
level=info ts=2026-05-09T19:56:48.448873001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:56:48.45451234Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:56:48.650577701Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-09T19:56:49.015464358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:56:51.94358226Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-09T19:56:52.143945332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:56:52.151944137Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T19:56:53.347135419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:56:53.350338462Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:56:53.350356541Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:56:53.350377386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:56:53.353177581Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:56:54.015244263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:56:55.165702906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:56:55.16832596Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:56:55.236681993Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-09T19:56:55.349982718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:56:55.356230518Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-09T19:56:55.879297743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:56:56.852111495Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:56:57.637589793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:56:57.640630598Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:56:58.530039793Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-09T19:56:58.530059902Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=4
level=warn ts=2026-05-09T19:56:59.015806841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:57:01.467385709Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:57:01.82152382Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-09T19:57:03.997751746Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T19:57:03.998193761Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:57:04.0155376Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:57:04.015555349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:57:04.015563107Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:57:04.015552465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:57:05.116954873Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-09T19:57:05.11708533Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=2
level=info ts=2026-05-09T19:57:05.427236499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:57:05.433063754Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:57:06.658854743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:57:06.662525876Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-09T19:57:07.359568366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:43064"
level=debug ts=2026-05-09T19:57:08.407426115Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=warn ts=2026-05-09T19:57:09.016033674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:57:09.407980573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:57:09.411193414Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:57:11.700578336Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=warn ts=2026-05-09T19:57:14.015442794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:57:14.465769527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:36204"
level=info ts=2026-05-09T19:57:14.68181693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:57:14.685512215Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T19:57:14.992255343Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-09T19:57:16.533651165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:57:16.536821145Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:57:16.831648846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:57:16.840446456Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T19:57:18.285704566Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-09T19:57:18.28572752Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-09T19:57:19.015494357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:57:20.378788728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:57:20.382414936Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T19:57:21.577424024Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=info ts=2026-05-09T19:57:22.077011342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:57:22.082997001Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:57:22.541217188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:57:22.544218838Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T19:57:24.015195803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:57:24.869944828Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-09T19:57:25.88172971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T19:57:26.217551028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:57:26.222397909Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:57:28.163067963Z caller=checkpoint.go:372 msg="writing series" size="615 kB"
level=warn ts=2026-05-09T19:57:29.015933238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:57:31.455758043Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:57:32.198034716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:57:32.204324919Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:57:34.015258458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:57:34.015284007Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:57:34.015291395Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:57:34.015321412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:57:34.235811743Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:57:34.235859534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T19:57:34.239325669Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T19:57:34.748295524Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-09T19:57:35.111489232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:57:35.116919814Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T19:57:36.929375911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:57:36.93424265Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T19:57:37.655218073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:57:37.658564312Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T19:57:38.041171357Z caller=checkpoint.go:372 msg="writing series" size="344 kB"
level=debug ts=2026-05-09T19:57:38.041192373Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-09T19:57:39.01585324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:57:39.323113129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:57:39.330192413Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:57:41.333643935Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-09T19:57:42.258974446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:57:44.016027259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:57:44.46880711Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:52524"
level=debug ts=2026-05-09T19:57:44.62645225Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-09T19:57:47.919356787Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T19:57:49.015409733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:57:50.018283879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:57:50.023298391Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T19:57:50.36133516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T19:57:50.364299103Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T19:57:51.21214413Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T19:57:51.624207489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:57:51.627221207Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:57:52.342453339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:57:52.345501426Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T19:57:54.015348447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:57:54.521783488Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-09T19:57:54.52181002Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=5
level=info ts=2026-05-09T19:57:54.618900723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:57:54.621891722Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-05-09T19:57:55.883898925Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:57:57.798037585Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-09T19:57:57.798061998Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=warn ts=2026-05-09T19:57:59.016068416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:58:01.089750064Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-09T19:58:01.535963344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:58:01.540253679Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:58:02.194530882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:58:02.197672058Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:58:03.997403613Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:58:03.99845435Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:58:04.015809796Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:58:04.015832343Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:58:04.015841525Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:58:04.015891141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:58:04.112573489Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:58:04.382943665Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-09T19:58:04.571867757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:58:04.574669476Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:58:04.574686687Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:58:04.574705037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:58:04.577255188Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T19:58:05.772178978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:58:05.79652856Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T19:58:06.077530852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:58:06.080563474Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T19:58:06.40350204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:58:06.406997491Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-09T19:58:07.365425004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:40644"
level=debug ts=2026-05-09T19:58:07.676322167Z caller=checkpoint.go:372 msg="writing series" size="626 kB"
level=info ts=2026-05-09T19:58:07.89463772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:58:07.898875569Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T19:58:09.015615025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:58:10.968598868Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-09T19:58:13.884189373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:58:13.891324311Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:58:14.015627432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:58:14.26172072Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:58:14.26174392Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-09T19:58:17.553470172Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-09T19:58:19.015578401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:58:20.545585625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:58:20.551454179Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:58:20.846414739Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-09T19:58:22.113979702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:58:22.118067705Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T19:58:24.015240304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:58:24.138776037Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-09T19:58:24.463304392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:58:24.466614952Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T19:58:25.886428253Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T19:58:27.31414386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T19:58:27.317568366Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T19:58:27.431305695Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=info ts=2026-05-09T19:58:28.081827718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:58:28.089999087Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T19:58:29.016172267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:58:30.724531213Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-09T19:58:34.01527955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:58:34.015304492Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:58:34.015312748Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:58:34.01531026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:58:34.01638919Z caller=checkpoint.go:384 msg="flushing series" totalSize="849 kB" series=5
level=info ts=2026-05-09T19:58:34.024312879Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-09T19:58:34.026915214Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010752723s
level=info ts=2026-05-09T19:58:34.047550804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:58:34.052629819Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T19:58:36.244480626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:58:36.247680852Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T19:58:39.007110023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:58:39.011783297Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T19:58:39.015891441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:58:39.803087639Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:58:40.167800764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:58:40.170913402Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:58:41.987834467Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T19:58:42.219828915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:58:42.223184699Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-09T19:58:42.223202816Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:58:42.223225122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:58:42.225575531Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T19:58:44.015654967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:58:46.514892462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T19:58:46.518543534Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:58:47.214681461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:58:47.218015353Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T19:58:49.016148477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:58:50.86853208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:58:50.872123324Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T19:58:51.180995742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39810"
level=info ts=2026-05-09T19:58:52.174602992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="122 MB" avg_uncomp="20 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:58:52.18111006Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T19:58:54.016071379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:58:54.605992331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:58:54.609150334Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T19:58:54.915729216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:58:54.919072719Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T19:58:55.889583582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=warn ts=2026-05-09T19:58:59.015549806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:59:00.223140307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:59:00.226366326Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:59:00.226386171Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:59:00.226408617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:59:00.229066759Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T19:59:01.529574467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:59:01.532843431Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T19:59:01.604195641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:59:01.609573906Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T19:59:03.997295892Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T19:59:03.998357071Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T19:59:04.01571799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:59:04.015735385Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:59:04.015742617Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:59:04.015758147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:59:04.015743657Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T19:59:04.016040305Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-09T19:59:04.016049869Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T19:59:04.016070114Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.159µs success=true
level=debug ts=2026-05-09T19:59:04.016365561Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-09T19:59:06.842910319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T19:59:06.849226134Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T19:59:07.269597098Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-09T19:59:07.371042992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47788"
level=info ts=2026-05-09T19:59:08.624297434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:59:08.630865554Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T19:59:09.015715299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:59:09.526336626Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T19:59:09.526392821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="12 kB" avg_uncomp="12 kB" idle=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T19:59:09.530902306Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T19:59:10.522618308Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T19:59:11.430686947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:59:11.433722675Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T19:59:13.775888733Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=warn ts=2026-05-09T19:59:14.015351641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:59:17.029329341Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T19:59:17.093562187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T19:59:17.096933011Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T19:59:19.016132824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:59:19.694233064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T19:59:19.701298427Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T19:59:20.282543784Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T19:59:20.282565744Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T19:59:21.374304485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:59:21.377755466Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-09T19:59:21.377773314Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:59:21.377799131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:59:21.380238307Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T19:59:23.535530793Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=info ts=2026-05-09T19:59:23.55932742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T19:59:23.562535969Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=info ts=2026-05-09T19:59:23.59488643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T19:59:23.597519623Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T19:59:24.015586341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T19:59:25.892193241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:59:26.787646256Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-09T19:59:26.788397024Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T19:59:29.015815669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:59:29.295140672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T19:59:29.298532172Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T19:59:30.041558246Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T19:59:30.305015931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:59:30.305449995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:59:30.308381888Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:59:30.308398285Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=error ts=2026-05-09T19:59:30.308391724Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T19:59:30.308424144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:59:30.310852809Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:59:32.177120898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:59:32.180015092Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T19:59:32.998908411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:59:33.001924683Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:59:33.00194318Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:59:33.001962592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:59:33.004258022Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:59:33.29407481Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T19:59:34.016024505Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T19:59:34.016051962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T19:59:34.01606057Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T19:59:34.016065536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:59:36.547349579Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-09T19:59:37.374692407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:54672"
level=warn ts=2026-05-09T19:59:39.01558395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:59:39.800072683Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T19:59:39.800112649Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-09T19:59:41.573509292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T19:59:41.579377972Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T19:59:42.952503838Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T19:59:43.053023271Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-09T19:59:44.015953482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:59:44.937384056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T19:59:44.940910339Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T19:59:45.983977226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T19:59:45.987261728Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T19:59:46.306498976Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-09T19:59:49.015433586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:59:49.558996082Z caller=checkpoint.go:372 msg="writing series" size="646 kB"
level=info ts=2026-05-09T19:59:50.231146129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T19:59:50.234346042Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T19:59:50.268432289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T19:59:50.271670627Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-09T19:59:51.184844261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54974"
level=info ts=2026-05-09T19:59:51.522585283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T19:59:51.529424048Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T19:59:52.812187564Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-09T19:59:53.295488356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:59:53.298866368Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:59:53.29888648Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T19:59:53.298908096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T19:59:53.301399531Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T19:59:53.684420951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T19:59:53.687546705Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-09T19:59:54.015250357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T19:59:55.79585927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="592 B" avg_comp="592 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T19:59:55.799171149Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T19:59:55.893633857Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T19:59:56.06497516Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-09T19:59:58.644676234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T19:59:58.650968389Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T19:59:58.750879968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T19:59:58.756278708Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T19:59:59.015977636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T19:59:59.320025687Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T19:59:59.320047995Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=6
level=info ts=2026-05-09T20:00:00.050883306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:00:00.053869421Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:00:02.571034397Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=info ts=2026-05-09T20:00:03.997519957Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:00:03.997789442Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=debug ts=2026-05-09T20:00:04.000106643Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T20:00:04.000148089Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975951 n=1
level=debug ts=2026-05-09T20:00:04.000189162Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975951 n=1
level=info ts=2026-05-09T20:00:04.016126422Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:00:04.01614686Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:00:04.016155193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:00:04.016183862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:00:05.759287883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:00:05.763064045Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:00:05.823878076Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-09T20:00:06.946655972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:00:06.949457776Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-09T20:00:07.377792701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:52604"
level=debug ts=2026-05-09T20:00:07.98212259Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:00:07.983547303Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T20:00:08.912491076Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T20:00:09.015927409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:00:09.06206432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:00:09.064934713Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T20:00:09.136987973Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-05-09T20:00:09.137016803Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=3
level=debug ts=2026-05-09T20:00:12.330310047Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-09T20:00:14.015399876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:00:14.481350418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:34740"
level=info ts=2026-05-09T20:00:15.42508249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:00:15.431507739Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:00:15.583014751Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T20:00:16.492835238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:00:16.496188707Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T20:00:18.047676536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:00:18.056868889Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:00:18.836194267Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-09T20:00:19.015915614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:00:20.719893156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:00:20.724489571Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T20:00:21.061340504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:00:21.065169441Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:00:21.065194663Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:00:21.065215612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:00:21.068120439Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-09T20:00:21.186014741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37992"
level=debug ts=2026-05-09T20:00:22.089984971Z caller=checkpoint.go:372 msg="writing series" size="388 kB"
level=info ts=2026-05-09T20:00:23.450632321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:00:23.454516226Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T20:00:24.01578538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:00:25.342706498Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
ts=2026-05-09T20:00:25.894651085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T20:00:26.242545597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:00:26.246127969Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:00:28.594939837Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-09T20:00:29.016008532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:00:30.349715277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:00:30.352946567Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:00:30.352966218Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:00:30.352988271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:00:30.355678421Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:00:31.588364895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:00:31.591918942Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T20:00:31.848829699Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=debug ts=2026-05-09T20:00:31.848851183Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-09T20:00:34.015156737Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:00:34.015161246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:00:34.015191199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:00:34.015199365Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T20:00:35.101115984Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-05-09T20:00:36.038788279Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:00:38.3548154Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-09T20:00:38.592511822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T20:00:38.596559558Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-09T20:00:39.015551621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:00:40.01413502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:00:40.016964432Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T20:00:41.607707675Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-05-09T20:00:42.108227319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:00:42.115912192Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T20:00:44.015703668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:00:44.542790982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:00:44.546185907Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T20:00:44.862059693Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T20:00:44.862081053Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-09T20:00:44.867482731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:00:44.870944242Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=info ts=2026-05-09T20:00:45.529285461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:00:45.53949751Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:00:48.113548522Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-09T20:00:48.284938095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:00:48.28807841Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T20:00:49.015942212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:00:49.258953824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:00:49.26198911Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T20:00:51.18787516Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38812"
level=debug ts=2026-05-09T20:00:51.36617574Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T20:00:51.747218658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:00:51.751815938Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:00:52.520003374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:00:52.526253269Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:00:52.526275502Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:00:52.526302079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:00:52.532321498Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:00:52.997756511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:00:53.000772961Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T20:00:54.015227418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:00:54.619086206Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
ts=2026-05-09T20:00:55.896856426Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T20:00:57.872558615Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-09T20:00:59.015924958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:00:59.131137671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:00:59.134899951Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T20:01:00.769407848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:01:00.773227356Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:01:01.1256377Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T20:01:03.997695339Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:01:03.998175763Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:01:04.015568309Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:01:04.015593424Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:01:04.015601948Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:01:04.015611327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:01:04.378277008Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=debug ts=2026-05-09T20:01:04.928269946Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:01:07.631424633Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T20:01:08.08770987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:01:08.091324495Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:01:08.234713723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:01:08.241086993Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:01:08.884701816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:01:08.887921432Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T20:01:09.01604968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:01:10.072273262Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:01:10.88499665Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T20:01:14.01525107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:01:14.138394625Z caller=checkpoint.go:372 msg="writing series" size="934 kB"
level=debug ts=2026-05-09T20:01:14.138416244Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=9
level=debug ts=2026-05-09T20:01:17.390612992Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-09T20:01:18.020056993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:01:18.030470662Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:01:18.030498968Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:01:18.030547483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:01:18.038427608Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:01:18.438550647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:01:18.44186396Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:01:18.441888077Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:01:18.441911267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:01:18.444726834Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T20:01:19.016029015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:01:20.35154445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T20:01:20.354616653Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T20:01:20.376678702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:01:20.380043829Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:01:20.419346434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:01:20.421998203Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:01:20.656486426Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T20:01:20.656512095Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=debug ts=2026-05-09T20:01:23.896591445Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T20:01:24.016088947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:01:24.404381934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:01:24.408386692Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T20:01:25.900249772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T20:01:27.150117165Z caller=checkpoint.go:372 msg="writing series" size="625 kB"
level=info ts=2026-05-09T20:01:27.719320947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:01:27.722803457Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:01:29.015702392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:01:29.098011265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:01:29.105491603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:01:30.402131243Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T20:01:32.044928061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:01:32.053267928Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:01:32.915612077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:01:32.918963446Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T20:01:33.656265304Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T20:01:33.656287614Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=4
level=info ts=2026-05-09T20:01:34.015271152Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:01:34.015303492Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:01:34.01531405Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:01:34.015306943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:01:34.265630654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:01:34.269552321Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T20:01:35.946449657Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:01:36.908689473Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
ts=2026-05-09T20:01:37.383723371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:53404"
level=warn ts=2026-05-09T20:01:39.01591278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:01:39.223314238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:01:39.227226143Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:01:39.810835669Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:01:40.161262146Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-09T20:01:40.780936924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:01:40.784136641Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:01:41.65188017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:01:41.658259123Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:01:41.65828022Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:01:41.658306235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:01:41.664730671Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:01:43.417212539Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T20:01:43.417234224Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=3
level=warn ts=2026-05-09T20:01:44.016164343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:01:46.113122198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:01:46.119816722Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:01:46.557756744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:01:46.561907526Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:01:46.667294112Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=info ts=2026-05-09T20:01:47.583920819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:01:47.587308115Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T20:01:49.016209224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:01:49.716369599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:01:49.72358066Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:01:49.920850743Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-09T20:01:51.192999777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54438"
level=debug ts=2026-05-09T20:01:53.173511723Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-09T20:01:53.769415683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:01:53.776785256Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T20:01:54.015214393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:01:55.630213308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:01:55.634759058Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-09T20:01:55.902841903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:01:56.426243229Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=warn ts=2026-05-09T20:01:59.015562229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:01:59.682374922Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-09T20:01:59.682398642Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=5
level=info ts=2026-05-09T20:01:59.985887879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:01:59.98960801Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T20:02:02.614545354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:02:02.617824613Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:02:02.74714948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T20:02:02.750201763Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-09T20:02:02.932879522Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=info ts=2026-05-09T20:02:03.713142763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:02:03.716671793Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:02:03.716692111Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:02:03.716712849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:02:03.719155447Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:02:03.997521491Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:02:03.997907024Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:02:04.015182979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:02:04.015209636Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:02:04.015217397Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:02:04.015346374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:02:06.187197242Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-09T20:02:06.187219665Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=2
level=debug ts=2026-05-09T20:02:06.921592996Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:02:08.241572858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:02:08.248282166Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T20:02:09.015546203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:02:09.439147749Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-09T20:02:10.833162223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:02:10.836459165Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:02:11.878226356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:02:11.881534548Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:02:12.691876612Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=warn ts=2026-05-09T20:02:14.016033891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:02:15.944715859Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=warn ts=2026-05-09T20:02:19.015971874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:02:19.198309789Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-09T20:02:19.198330178Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-05-09T20:02:19.661072763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:02:19.66405193Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:02:21.951471329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T20:02:22.126140655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:02:22.133239887Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:02:22.450399884Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-09T20:02:23.598430589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:02:23.601660348Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:02:23.712008549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:02:23.715003601Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T20:02:24.016066736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:02:25.704048255Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-09T20:02:25.905166754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T20:02:27.49258938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:02:27.501012908Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:02:28.957017816Z caller=checkpoint.go:372 msg="writing series" size="486 kB"
level=warn ts=2026-05-09T20:02:29.015846444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:02:30.85069829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:02:30.856672835Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:02:32.209936979Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T20:02:34.015495319Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:02:34.015523549Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:02:34.015531111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:02:34.015524792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:02:35.279262534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:02:35.282667868Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:02:35.46237751Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-09T20:02:37.966909681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:02:37.970205626Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T20:02:38.715863459Z caller=checkpoint.go:372 msg="writing series" size="385 kB"
level=info ts=2026-05-09T20:02:39.001850764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:02:39.004899311Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T20:02:39.015959774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:02:41.810319311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:02:41.968599023Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T20:02:43.314369598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:02:43.318259332Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T20:02:44.015236165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:02:45.221576458Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-09T20:02:46.437121555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:02:46.440838475Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:02:47.486769576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:02:47.490115769Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:02:47.53916287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:02:47.542546699Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T20:02:48.474207345Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T20:02:49.015884698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:02:51.403123803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:02:51.409695201Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T20:02:51.727720161Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-09T20:02:51.727741319Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-09T20:02:53.225160731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T20:02:53.228706352Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-09T20:02:54.015779127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:02:55.001624474Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-09T20:02:55.001649481Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=1
level=info ts=2026-05-09T20:02:55.050348712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:02:55.053754288Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-09T20:02:55.907168848Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:02:58.23474988Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T20:02:58.234772012Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=1
level=warn ts=2026-05-09T20:02:59.015719099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:02:59.664680173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:02:59.669843084Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:03:01.487113352Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-09T20:03:03.997538483Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:03:03.99776118Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:03:04.016138559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:03:04.01615786Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T20:03:04.016153642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:03:04.016165064Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:03:04.354654629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:03:04.361584908Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:03:04.739278493Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-09T20:03:06.132009298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:03:06.135369861Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:03:07.621458716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:03:07.629546309Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:03:07.993053922Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=debug ts=2026-05-09T20:03:08.869036997Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T20:03:09.015398464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:03:10.529759982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:03:10.532637986Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T20:03:10.811231894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:03:11.245674624Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=info ts=2026-05-09T20:03:12.486969903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:03:12.490236921Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T20:03:14.015643495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:03:14.196121704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:03:14.201977155Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:03:14.49918986Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T20:03:14.49921179Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-09T20:03:15.561875444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:03:15.569130613Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:03:15.569150954Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:03:15.569179088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:03:15.575604421Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:03:17.752004679Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-09T20:03:19.015938341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:03:21.004779156Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
ts=2026-05-09T20:03:21.200758916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59322"
level=warn ts=2026-05-09T20:03:24.015997423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:03:24.258166677Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-09T20:03:25.750958494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:03:25.757884323Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:03:25.757906154Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:03:25.757930671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:03:25.763075939Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T20:03:25.909796286Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:03:27.511312007Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=warn ts=2026-05-09T20:03:29.015786412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:03:29.212132897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:03:29.218203419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:03:30.76353666Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-09T20:03:33.518844034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:03:33.523260889Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T20:03:34.015683742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:03:34.015692553Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:03:34.015718575Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:03:34.015726715Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T20:03:34.016786578Z caller=checkpoint.go:384 msg="flushing series" totalSize="857 kB" series=5
level=info ts=2026-05-09T20:03:34.022498388Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-09T20:03:34.023803275Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00756378s
level=info ts=2026-05-09T20:03:35.265983033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:03:35.269631075Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T20:03:36.235559206Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:03:38.048795605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:03:38.051979575Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T20:03:39.015554341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:03:40.21943964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:03:40.222270623Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:03:40.698084876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:03:40.70150288Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:03:43.115494587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:03:43.118602745Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T20:03:44.01518363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:03:44.502680085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:55562"
level=info ts=2026-05-09T20:03:44.823259862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:03:44.826113434Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T20:03:46.411419476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:03:46.414815028Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:03:47.623142137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:03:47.626223904Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=info ts=2026-05-09T20:03:47.70546484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:03:47.708479974Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:03:48.374053986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T20:03:48.377238988Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-09T20:03:49.015699075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:03:51.202041229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39590"
level=info ts=2026-05-09T20:03:52.033939098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:03:52.040449349Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:03:52.807667161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:03:52.815170525Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T20:03:54.015769662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:03:55.731674133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:03:55.73339802Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T20:03:55.912236825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T20:03:57.003270661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:03:57.008801009Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:03:58.621958116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:03:58.625196935Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T20:03:59.016032117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:04:00.19558726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:04:00.202455856Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:04:03.587800897Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:04:03.587845227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="510 kB" avg_comp="510 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:04:03.593185301Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:04:03.997512588Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:04:03.997868129Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:04:04.015156885Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:04:04.015172059Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:04:04.015178872Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:04:04.015186895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:04:04.016226813Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T20:04:04.016348342Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-09T20:04:04.016356688Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T20:04:04.016373692Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.385µs success=true
level=debug ts=2026-05-09T20:04:04.016629736Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-09T20:04:07.270165851Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-09T20:04:07.397751574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47886"
level=info ts=2026-05-09T20:04:08.758923163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:04:08.762214707Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T20:04:09.015704043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:04:10.523345761Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T20:04:11.699007874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:04:11.702114424Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:04:13.776255716Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=warn ts=2026-05-09T20:04:14.016180994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:04:14.505622404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:49010"
level=info ts=2026-05-09T20:04:16.64892349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:04:16.652466872Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:04:16.652484501Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:04:16.652504975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:04:16.655069018Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:04:17.029298674Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-09T20:04:19.015848842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:04:19.219922636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:04:19.223636811Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:04:19.618131963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:04:19.621737753Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:04:20.282203542Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T20:04:20.282226047Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T20:04:21.678036838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:04:21.681134585Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:04:23.535229425Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=info ts=2026-05-09T20:04:23.859214833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:04:23.863224355Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T20:04:24.015678947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:04:25.913373679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:04:26.788140455Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-05-09T20:04:29.016111383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:04:29.335252693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T20:04:29.338475743Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-09T20:04:29.510212894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:04:29.513157559Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:04:30.041086778Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-09T20:04:30.386008222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:04:30.389417176Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:04:30.559960925Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:04:31.344213152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:04:31.347428189Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T20:04:33.294056903Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T20:04:33.768308578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:04:33.772806095Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:04:33.976485354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:04:33.979521021Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T20:04:34.008700287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:04:34.011492881Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:04:34.015593599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:04:34.015605547Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:04:34.015613091Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:04:34.015638018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:04:36.547273212Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-09T20:04:39.015745695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:04:39.800762587Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T20:04:39.800782372Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-09T20:04:41.553934256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:04:41.561664594Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:04:43.052952025Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-09T20:04:43.859238812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:04:43.865580954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T20:04:44.015985638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:04:44.314214281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:04:44.320039538Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:04:45.795141573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="592 B" avg_comp="592 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:04:45.798083746Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:04:45.798116709Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:04:45.798137275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:04:45.800557137Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:04:45.945847587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:04:45.94868604Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:04:46.30593866Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-09T20:04:49.015452587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:04:49.559776663Z caller=checkpoint.go:372 msg="writing series" size="529 kB"
level=debug ts=2026-05-09T20:04:52.812549625Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-09T20:04:54.016123006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:04:54.673835257Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:04:55.669551709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:04:55.676211912Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T20:04:55.916202814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:04:56.065361659Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=debug ts=2026-05-09T20:04:56.243134525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T20:04:58.831144207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:04:58.834043284Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T20:04:59.015216441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:04:59.193413573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:04:59.197495Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:04:59.320122674Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T20:04:59.320143558Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=6
level=info ts=2026-05-09T20:04:59.642847746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:04:59.645591254Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:05:00.087462059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:05:00.090602065Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:05:01.292133963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:05:01.295305621Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:05:02.570950465Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-09T20:05:03.997708135Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:05:03.998112158Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:05:04.015419349Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:05:04.015444103Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:05:04.015452455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:05:04.015448635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:05:04.720296934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:05:04.724650062Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:05:05.824195216Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-09T20:05:07.471110765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:05:07.47434561Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T20:05:09.015427021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:05:09.136441122Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-05-09T20:05:09.136469164Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=debug ts=2026-05-09T20:05:12.329986578Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-09T20:05:14.015879133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:05:15.583703254Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T20:05:16.899472165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:05:16.902592042Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:05:17.329810756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:05:17.332675328Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-09T20:05:17.332692217Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:05:17.332712108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:05:17.335037212Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T20:05:18.159421743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-09T20:05:18.162391Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e0e098575%3A19e0e098576%3Aa72c486f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-09T20:05:18.1624082Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:05:18.162432636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="738 kB" avg_comp="738 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:05:18.165823376Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T20:05:18.836793563Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-09T20:05:19.016150471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:05:20.535180618Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:05:22.089701008Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=warn ts=2026-05-09T20:05:24.015765599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:05:24.590435323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:05:24.594691329Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T20:05:25.342196093Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-09T20:05:25.446207189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:05:25.449201931Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-09T20:05:25.919521779Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T20:05:27.983767254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:05:27.986961032Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:05:28.594960699Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-09T20:05:29.015512696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:05:30.317422982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:05:30.326005176Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:05:31.848470545Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-09T20:05:31.976406852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:05:31.979494416Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T20:05:33.289556242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:05:33.292574154Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:05:34.015574886Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:05:34.015599743Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:05:34.015607566Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:05:34.015605725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:05:35.101463193Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=info ts=2026-05-09T20:05:35.411346372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:05:35.420435672Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:05:35.61100832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:05:35.614139871Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:05:38.016145166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:05:38.020249691Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T20:05:38.354178545Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-09T20:05:38.354202302Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-09T20:05:39.015391826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:05:41.412198726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:05:41.41637292Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:05:41.607564497Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-09T20:05:42.065797133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:05:42.071869282Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:05:42.465039789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:05:42.468017705Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:05:42.468036792Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:05:42.468060972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:05:42.470490568Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:05:44.015845272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:05:44.862195291Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T20:05:44.862216244Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=debug ts=2026-05-09T20:05:48.113565155Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-09T20:05:49.015823209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:05:49.922537053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:05:49.925912883Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:05:50.94171185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:05:50.949193014Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T20:05:51.213378266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:41064"
level=debug ts=2026-05-09T20:05:51.366466908Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T20:05:52.612502166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:05:52.615959265Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:05:52.6812013Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T20:05:54.015194416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:05:54.618999151Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
ts=2026-05-09T20:05:55.922025785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T20:05:56.162557928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:05:56.165763932Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=info ts=2026-05-09T20:05:56.483721709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:05:56.48702729Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T20:05:56.725141106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:05:57.872450952Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-09T20:05:59.015658985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:06:00.910391224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:06:00.913520476Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:06:01.125240329Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T20:06:03.125255165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:06:03.128725168Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:06:03.997512303Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:06:03.997823193Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=warn ts=2026-05-09T20:06:04.015170055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:06:04.015172912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:06:04.015191222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:06:04.015199152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T20:06:04.379088973Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-09T20:06:05.805117049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:06:05.809394802Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T20:06:07.63175746Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-09T20:06:07.895635332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:06:07.899406276Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:06:07.899425488Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:06:07.89944574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:06:07.901753026Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T20:06:09.015470202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:06:10.885244442Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T20:06:14.015846252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:06:14.138043299Z caller=checkpoint.go:372 msg="writing series" size="778 kB"
level=debug ts=2026-05-09T20:06:14.138069973Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
level=info ts=2026-05-09T20:06:15.480123068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:06:15.483569616Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:06:16.006365663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:06:16.010668067Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:06:16.412730693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:06:16.41624344Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:06:16.416262313Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:06:16.416282351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:06:16.418880137Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:06:17.390755556Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-09T20:06:18.532696884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:06:18.539920559Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T20:06:19.016146433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:06:19.503361965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:06:19.506409421Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:06:19.506433405Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:06:19.506456671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:06:19.508980087Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:06:20.655415032Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T20:06:20.655443414Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
ts=2026-05-09T20:06:21.21529598Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52168"
level=debug ts=2026-05-09T20:06:23.741033073Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:06:23.896717909Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T20:06:24.015949315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:06:25.924789669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T20:06:26.758790706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:06:26.765346011Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:06:27.149854784Z caller=checkpoint.go:372 msg="writing series" size="643 kB"
level=info ts=2026-05-09T20:06:27.680089553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:06:27.683144499Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:06:28.386052056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:06:28.389126435Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:06:29.015661818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:06:29.149958533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:06:29.153579416Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:06:29.672856771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:06:29.679632922Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:06:30.403155857Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-05-09T20:06:32.287797532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:06:32.290821358Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:06:32.760976977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:06:32.764671727Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:06:33.136887612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:06:33.139737316Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:06:33.655791582Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-09T20:06:33.655814592Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=warn ts=2026-05-09T20:06:34.01611716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:06:34.01611834Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:06:34.016143749Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:06:34.016153787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:06:34.541569355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:06:34.544476798Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:06:36.909225998Z caller=checkpoint.go:372 msg="writing series" size="612 kB"
ts=2026-05-09T20:06:37.411264872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:51076"
level=info ts=2026-05-09T20:06:38.023667388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:06:38.030637031Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T20:06:39.015878256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:06:39.335924248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:06:39.339178283Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:06:40.161871597Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-09T20:06:42.870900167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:06:43.417047192Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-09T20:06:43.41706829Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=3
level=warn ts=2026-05-09T20:06:44.016182067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:06:44.515369796Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:47972"
level=info ts=2026-05-09T20:06:45.404985199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:06:45.408589455Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T20:06:46.627857945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:06:46.630673309Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T20:06:46.668006555Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-09T20:06:49.016163196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:06:49.026295568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:06:49.029841126Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:06:49.32649719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:06:49.329275337Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:06:49.329293228Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:06:49.329314047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:06:49.331459808Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T20:06:49.920593284Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-09T20:06:51.72026741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:06:51.720284846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="174 kB" avg_comp="174 kB" total_uncomp="892 kB" avg_uncomp="892 kB" idle=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:06:51.723665726Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=debug ts=2026-05-09T20:06:53.173861868Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=info ts=2026-05-09T20:06:53.29947969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:06:53.302540103Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T20:06:54.015689125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:06:54.814012444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:06:54.816913281Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
ts=2026-05-09T20:06:55.928051524Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:06:56.427151987Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-09T20:06:58.839134372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:06:58.84568978Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T20:06:59.016042905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:06:59.240986483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:06:59.245465501Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:06:59.682542347Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-09T20:06:59.68256639Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=5
level=info ts=2026-05-09T20:06:59.788617358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:06:59.791583124Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:07:00.464030048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:07:00.467239818Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:07:02.932614324Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-09T20:07:03.99777456Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:07:03.998189597Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:07:04.015555736Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:07:04.015577558Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:07:04.015585451Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:07:04.015595819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:07:06.188062478Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-09T20:07:06.188085036Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
ts=2026-05-09T20:07:07.414810066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47242"
level=info ts=2026-05-09T20:07:07.584422872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:07:07.588196267Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:07:08.098831508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:07:08.105220024Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T20:07:09.015664569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:07:09.148963443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:07:09.152594129Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=debug ts=2026-05-09T20:07:09.43920869Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=info ts=2026-05-09T20:07:11.027392014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:07:11.033691221Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:07:11.567214062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:07:11.573330099Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:07:11.573348517Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:07:11.573372908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:07:11.579177948Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:07:12.692211582Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=info ts=2026-05-09T20:07:12.835117772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:07:12.838556831Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:07:14.016069249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:07:14.518791245Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39254"
level=debug ts=2026-05-09T20:07:15.944562314Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=warn ts=2026-05-09T20:07:19.01575423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:07:19.19726076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T20:07:19.197860145Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-05-09T20:07:19.197878036Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=error ts=2026-05-09T20:07:19.200715997Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:07:21.064166673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:07:21.093318428Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:07:21.093349286Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:07:21.093382885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:07:21.100661688Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-09T20:07:21.220833995Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53884"
level=debug ts=2026-05-09T20:07:22.450859881Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=info ts=2026-05-09T20:07:23.027677918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:07:23.030912995Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:07:23.030934084Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:07:23.030957144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:07:23.03373416Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T20:07:24.015973168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:07:25.70423779Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-09T20:07:25.931196594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T20:07:26.529931452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:07:26.536420855Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:07:27.200558144Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:07:27.428082576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:07:27.431373939Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T20:07:28.956879019Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=warn ts=2026-05-09T20:07:29.016224325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:07:29.451414265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:07:29.455806793Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:07:30.539300515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:07:30.542520683Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T20:07:31.148231568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:07:31.152022682Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:07:32.135965759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:07:32.143579613Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:07:32.209923042Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T20:07:34.015275694Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:07:34.015286134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:07:34.015300008Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:07:34.015310205Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:07:34.38090881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:07:34.384424674Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:07:34.384449224Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:07:34.384475964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:07:34.387270949Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T20:07:35.083469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:07:35.08646396Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T20:07:35.462610302Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-09T20:07:36.383221415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:07:36.386272967Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:07:38.71654018Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=warn ts=2026-05-09T20:07:39.016114186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:07:40.542155788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:07:41.968991222Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T20:07:42.05456843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:07:42.057364931Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T20:07:42.292138581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:07:42.294926397Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:07:42.513559895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:07:42.51676668Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=warn ts=2026-05-09T20:07:44.015745914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:07:44.297225296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:07:44.300462019Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:07:45.13749276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:07:45.140496582Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T20:07:45.221900445Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-09T20:07:48.474845864Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=info ts=2026-05-09T20:07:48.805002128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:07:48.80801317Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T20:07:49.015389286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:07:51.728485093Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-09T20:07:51.728506082Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=debug ts=2026-05-09T20:07:53.234868941Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T20:07:54.015757499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:07:54.282288829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:07:54.286390222Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:07:54.915528233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:07:54.92205673Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:07:54.922076398Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:07:54.922118398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:07:54.928797116Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:07:55.002698372Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-09T20:07:55.002722772Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=1
ts=2026-05-09T20:07:55.934199168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:07:58.234937389Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T20:07:58.234961066Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-09T20:07:59.01553391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:08:00.145391336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:08:00.152780382Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:08:01.181676286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:08:01.184689319Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T20:08:01.486728232Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-09T20:08:03.289063507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:08:03.295516345Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:08:03.9977602Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:08:03.99825Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:08:04.004518467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:08:04.007350734Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:08:04.01547857Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:08:04.015496321Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:08:04.015503925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:08:04.015505326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:08:04.739871099Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-09T20:08:04.794937346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:08:04.801492245Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:08:07.328548059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:08:07.331379204Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T20:08:07.421584412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:35814"
level=info ts=2026-05-09T20:08:07.822120453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:08:07.824964494Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:08:07.993825936Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=info ts=2026-05-09T20:08:08.854847714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:08:08.858377062Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:08:08.959562242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:08:08.966158512Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T20:08:09.016227894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:08:11.246440956Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=warn ts=2026-05-09T20:08:14.015669973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:08:14.499452815Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T20:08:14.499475727Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-09T20:08:16.819614119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:08:16.822558579Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T20:08:17.148353957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:08:17.149949694Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-09T20:08:17.751881996Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-09T20:08:19.015771733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:08:19.544929238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:08:19.55253849Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:08:19.552557819Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:08:19.552594765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:08:19.559448222Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:08:21.005573264Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-05-09T20:08:21.070635469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:08:21.074359768Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:08:21.472615835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:08:21.476493255Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:08:23.456139192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:08:23.459070051Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T20:08:24.016034649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:08:24.258228852Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-09T20:08:25.088590031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:08:25.091617983Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-09T20:08:25.936920281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:08:26.190629709Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:08:26.190638501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:08:26.193759071Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:08:27.511965103Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=warn ts=2026-05-09T20:08:29.016291763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:08:29.36264779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:08:29.366046637Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=info ts=2026-05-09T20:08:29.398543202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:08:29.401485558Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:08:29.401504203Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:08:29.401525641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:08:29.403852899Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:08:30.764078066Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=info ts=2026-05-09T20:08:33.060810696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:08:33.065144563Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:08:34.015162846Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:08:34.015187584Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:08:34.015195244Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:08:34.015212431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:08:34.017193532Z caller=checkpoint.go:384 msg="flushing series" totalSize="861 kB" series=5
level=info ts=2026-05-09T20:08:34.024378221Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-09T20:08:34.025858069Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009348059s
level=info ts=2026-05-09T20:08:36.497013507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:08:36.506304029Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T20:08:37.424950246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:44210"
level=info ts=2026-05-09T20:08:37.839447826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:08:37.843422853Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T20:08:39.016210762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:08:40.235146121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:08:40.239661062Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T20:08:40.659745454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:08:40.662734898Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:08:41.726975566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:08:41.729875667Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-09T20:08:44.015570446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:08:44.52720527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39458"
level=info ts=2026-05-09T20:08:46.317515675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:08:46.324589455Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:08:47.282141914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:08:47.285298076Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:08:47.364514956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:08:47.367358567Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T20:08:49.015961827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:08:50.84762586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:08:50.85101568Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:08:53.373202108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:08:53.376569439Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:08:54.015544235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:08:54.694840941Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:08:54.900246005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:08:54.906546133Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:08:55.142538539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:08:55.145531368Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-09T20:08:55.940069012Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T20:08:56.536406129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:08:56.539747383Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:08:57.712136398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:08:57.717523701Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T20:08:59.015998782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:09:03.653616361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:09:03.66019324Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-09T20:09:03.997528907Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:09:03.997985227Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:09:04.015273907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:09:04.015287871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T20:09:04.015283061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:09:04.015295134Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:09:04.016352994Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T20:09:04.01647232Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-09T20:09:04.016479891Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T20:09:04.016497739Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.934µs success=true
level=debug ts=2026-05-09T20:09:04.016753169Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-09T20:09:05.358394908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:09:05.361577755Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:09:07.269775874Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=debug ts=2026-05-09T20:09:08.564986964Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T20:09:09.016251133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:09:10.523516548Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T20:09:10.834467272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:09:10.837870074Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T20:09:11.687183424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:09:11.693625004Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:09:13.777047316Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=warn ts=2026-05-09T20:09:14.01545535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:09:14.530984622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:54018"
level=info ts=2026-05-09T20:09:14.7879305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:09:14.791058838Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:09:14.82576754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:09:14.82870389Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-09T20:09:17.029793728Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-09T20:09:19.015619468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:09:19.434832189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:09:19.438388704Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:09:20.28244328Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T20:09:20.282463855Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-09T20:09:20.5269809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:09:20.530497494Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:09:20.820371601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:09:20.823354549Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=debug ts=2026-05-09T20:09:23.535619281Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
level=warn ts=2026-05-09T20:09:24.015931305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:09:25.211447248Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T20:09:25.942123035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:09:26.788545719Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-09T20:09:27.658028145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:09:27.662025339Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T20:09:27.685114231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:09:27.68913257Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T20:09:29.015871124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:09:30.041507297Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-09T20:09:30.461827319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:09:30.465195342Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T20:09:33.294798114Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T20:09:34.015521042Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:09:34.015547625Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T20:09:34.015535013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:09:34.015556047Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:09:34.806453764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:09:34.814119208Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:09:34.877285968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:09:34.880300945Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:09:36.547329456Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-09T20:09:38.532954485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:09:38.539155579Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T20:09:39.015844584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:09:39.800365708Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-09T20:09:39.800390744Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-05-09T20:09:39.980080256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="592 B" avg_comp="592 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:09:39.983213541Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:09:41.592968539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:09:41.597339625Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=info ts=2026-05-09T20:09:42.811909959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:09:42.819138004Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:09:42.819157184Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:09:42.819183044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:09:42.825530051Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:09:43.053272714Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=info ts=2026-05-09T20:09:43.470392136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:09:43.4733699Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T20:09:44.015943463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:09:44.534467787Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:50828"
level=debug ts=2026-05-09T20:09:46.306308987Z caller=checkpoint.go:372 msg="writing series" size="8.5 kB"
level=warn ts=2026-05-09T20:09:49.015368371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:09:49.559780106Z caller=checkpoint.go:372 msg="writing series" size="632 kB"
level=info ts=2026-05-09T20:09:52.584602565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:09:52.587795468Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:09:52.812686903Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-09T20:09:52.863705591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:09:52.866716854Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:09:53.116550785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:09:53.119813132Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:09:53.73853504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:09:53.746079243Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T20:09:54.016037694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:09:55.944545897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:09:56.065710874Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-09T20:09:56.177062309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="356 kB" avg_comp="356 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" idle=1 labels="{stream=\"integration1-mfky\"}"
level=debug ts=2026-05-09T20:09:56.177161026Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-09T20:09:56.181369589Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:09:56.477318404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="342 kB" avg_comp="342 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" idle=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:09:56.4869482Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
level=info ts=2026-05-09T20:09:57.998207761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:09:58.001419326Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-09T20:09:59.015906571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:09:59.31999514Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T20:09:59.320014959Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=6
level=info ts=2026-05-09T20:09:59.56423307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:09:59.571689851Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:10:00.066674338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:10:00.069855613Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-09T20:10:02.571797025Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-09T20:10:03.997526544Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:10:03.997903705Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:10:04.015128974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:10:04.015152446Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:10:04.015184439Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:10:04.015202568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:10:05.824649367Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-09T20:10:05.858452913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T20:10:06.839512782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:10:06.843018663Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
ts=2026-05-09T20:10:07.434579108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:40552"
level=warn ts=2026-05-09T20:10:09.01596123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:10:09.134062449Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-09T20:10:09.134133374Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=info ts=2026-05-09T20:10:09.630615172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:10:09.633897025Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T20:10:10.187583422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:10:10.191008727Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:10:10.662239986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:10:10.666327479Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:10:10.666349493Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:10:10.666371297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:10:10.669789891Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:10:11.132149298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:10:11.135037944Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:10:11.318604456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:10:11.322854376Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T20:10:12.330937539Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-09T20:10:13.853149685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:10:13.859593639Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=warn ts=2026-05-09T20:10:14.01615107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:10:14.910755335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:10:14.914673531Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:10:15.462013506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:10:15.465277627Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:10:15.583793491Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T20:10:17.859595587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:10:17.862965195Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=debug ts=2026-05-09T20:10:18.836705181Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-09T20:10:19.016166948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:10:19.607538829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:10:19.614326922Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:10:22.09001597Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-09T20:10:22.434819349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:10:22.441878907Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T20:10:24.01522834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:10:24.212890542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:10:24.215769387Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=info ts=2026-05-09T20:10:24.949337262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:10:24.952433579Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T20:10:25.342625512Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
ts=2026-05-09T20:10:25.947324112Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:10:26.880895307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:10:28.595066047Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-09T20:10:29.015399473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:10:29.197464627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:10:29.20161153Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:10:30.025734459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:10:30.029185543Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:10:31.849407546Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=info ts=2026-05-09T20:10:33.480683721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:10:33.483737681Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:10:34.015210296Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:10:34.015237708Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:10:34.0152456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:10:34.0152976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:10:34.522518662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:10:34.526082214Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:10:35.101260957Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
level=info ts=2026-05-09T20:10:35.531303933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:10:35.535018402Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:10:36.790597661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:10:36.794787275Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:10:38.064279311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="145 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:10:38.071749052Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:10:38.3547671Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-09T20:10:38.354791587Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-09T20:10:39.016226571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:10:39.545459313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:10:39.549478628Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
level=debug ts=2026-05-09T20:10:41.608039139Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=info ts=2026-05-09T20:10:42.046290735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:10:42.04940526Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-09T20:10:44.016032292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:10:44.861817724Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T20:10:44.86184073Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-05-09T20:10:46.114362637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:10:46.11854602Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:10:48.113500054Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-09T20:10:49.015973539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:10:49.656372561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:10:49.66551175Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:10:51.06443202Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:10:51.194944361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:10:51.200587008Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:10:51.367008202Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T20:10:54.01608686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:10:54.619774681Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-09T20:10:54.669887839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:10:54.674109572Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T20:10:55.343271553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-09T20:10:55.949997512Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T20:10:56.116304473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:10:56.120153077Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:10:56.120179523Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:10:56.120212047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:10:56.123501743Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:10:57.143215897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:10:57.146306342Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T20:10:57.563402549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:10:57.566499999Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T20:10:57.872531421Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-09T20:10:59.015679968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:11:01.126132329Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-05-09T20:11:03.997464984Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:11:03.998514954Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T20:11:04.015899231Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:11:04.015922238Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:11:04.015930132Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:11:04.015946259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:11:04.378935538Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T20:11:05.528436041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:11:05.532046501Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T20:11:06.677879927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:11:06.684699432Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:11:07.631688685Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-09T20:11:09.016242045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:11:09.408346858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:11:09.413569964Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:11:09.465623507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:11:09.468758238Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:11:10.884417256Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-09T20:11:11.211522664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:11:11.215744408Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:11:11.680495673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:11:11.684185568Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:11:11.684205231Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:11:11.684226723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:11:11.686666132Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T20:11:14.015311676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:11:14.138191635Z caller=checkpoint.go:372 msg="writing series" size="791 kB"
level=debug ts=2026-05-09T20:11:14.138218456Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
ts=2026-05-09T20:11:14.54401276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:45874"
level=info ts=2026-05-09T20:11:15.703899571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:11:15.707009288Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=debug ts=2026-05-09T20:11:16.15928703Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:11:16.287738357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:11:16.290730785Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=info ts=2026-05-09T20:11:16.324791626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:11:16.327127549Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:11:16.496775785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:11:16.499633929Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:11:17.390483373Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-09T20:11:18.538064556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:11:18.543235595Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:11:18.642561731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:11:18.645363522Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:11:18.645383912Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:11:18.645405234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:11:18.647612166Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-09T20:11:19.015788516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:11:20.656006335Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T20:11:20.656027945Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
ts=2026-05-09T20:11:21.24532808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39142"
level=info ts=2026-05-09T20:11:21.621012875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:11:21.62603858Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
level=info ts=2026-05-09T20:11:23.541576988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:11:23.548137448Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:11:23.896447826Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T20:11:24.015582338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:11:24.629508403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:11:24.632498292Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:11:25.854649019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:11:25.85821472Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
ts=2026-05-09T20:11:25.953702819Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T20:11:26.611274009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:11:26.614159856Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T20:11:27.118534945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-09T20:11:27.121644329Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e0e098574%3A19e0e098581%3A9ded3b27?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-09T20:11:27.121663018Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:11:27.121685399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="451 kB" avg_comp="451 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:11:27.124598623Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T20:11:27.150002086Z caller=checkpoint.go:372 msg="writing series" size="703 kB"
level=warn ts=2026-05-09T20:11:29.015671821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:11:30.403142633Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-09T20:11:31.303212328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:11:31.306225138Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T20:11:33.656497221Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T20:11:33.656518082Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=4
level=info ts=2026-05-09T20:11:34.015835706Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:11:34.015861883Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:11:34.015869883Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:11:34.015859572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:11:34.829365572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:11:34.83353863Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:11:35.844143938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:11:35.847255374Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:11:36.655682613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:11:36.65965633Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:11:36.908644576Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
ts=2026-05-09T20:11:37.440909112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:43344"
level=warn ts=2026-05-09T20:11:39.01530321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:11:40.161740516Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-09T20:11:41.582593135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:11:41.589324335Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:11:41.589345104Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:11:41.589369053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:11:41.592979813Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:11:43.200596402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:11:43.203611583Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:11:43.416908451Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-09T20:11:43.416933162Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=3
level=warn ts=2026-05-09T20:11:44.015762721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:11:45.492726274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:11:45.496161221Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T20:11:45.716504732Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:11:46.668481394Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=debug ts=2026-05-09T20:11:46.713613265Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:11:48.794745178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:11:48.802591678Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T20:11:49.015881367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:11:49.921332418Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
ts=2026-05-09T20:11:51.246584819Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42672"
level=info ts=2026-05-09T20:11:51.80273948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:11:51.811170981Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:11:52.043888505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:11:52.046962382Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T20:11:53.174086267Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-09T20:11:54.015632236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:11:55.955868235Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:11:56.427151675Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-09T20:11:56.853431308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:11:56.857814683Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:11:57.074373345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:11:57.077324295Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:11:57.813489365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:11:57.816805783Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:11:57.816824177Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:11:57.816847217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:11:57.819036945Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:11:59.015364678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:11:59.682512847Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-09T20:11:59.682552947Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.3 MB" series=5
level=info ts=2026-05-09T20:12:00.496759355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:12:00.500408524Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=info ts=2026-05-09T20:12:02.275138589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:12:02.278386975Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T20:12:02.933361173Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=info ts=2026-05-09T20:12:03.458792001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:12:03.461884641Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:12:03.997550211Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:12:03.997555737Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:12:04.015985972Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:12:04.016001638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:12:04.016011491Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:12:04.016019953Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:12:04.085267936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:12:04.088634086Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T20:12:05.298204161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:12:05.301343222Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:12:06.186659566Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-09T20:12:06.186682624Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=2
level=info ts=2026-05-09T20:12:07.148563465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:12:07.151425277Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-09T20:12:07.443786854Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:35264"
level=info ts=2026-05-09T20:12:08.780550543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:12:08.786813743Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T20:12:09.015434334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:12:09.438704419Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=info ts=2026-05-09T20:12:09.50392685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:12:09.509641076Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:12:10.566401254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:12:10.569399892Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:12:10.829215004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:12:10.832031041Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:12:12.692545114Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=info ts=2026-05-09T20:12:13.09782121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:12:13.104904153Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-09T20:12:14.01562229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:12:14.298255424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:12:14.301383001Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
ts=2026-05-09T20:12:14.550758321Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:52624"
level=debug ts=2026-05-09T20:12:15.945615808Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T20:12:16.965499984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:12:16.968871061Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:12:18.522246629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:12:18.525706516Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
level=warn ts=2026-05-09T20:12:19.015923996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:12:19.19842343Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=debug ts=2026-05-09T20:12:19.198447079Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-09T20:12:19.462136711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:12:19.465776022Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:12:19.950170123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:12:22.450887094Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=warn ts=2026-05-09T20:12:24.015247678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:12:24.961416479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:12:24.966220723Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:12:25.653757989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:12:25.656799694Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:12:25.703967555Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-09T20:12:25.958418297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T20:12:26.347120718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:12:26.350939561Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:12:26.432252674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:12:26.435874463Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:12:28.95755136Z caller=checkpoint.go:372 msg="writing series" size="559 kB"
level=warn ts=2026-05-09T20:12:29.015554764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:12:32.210167307Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T20:12:34.015832981Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:12:34.015843188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:12:34.015867051Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:12:34.015877854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T20:12:34.942265693Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:12:34.943840858Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T20:12:35.173455342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:12:35.180724639Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:12:35.462740852Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-09T20:12:38.716116628Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=info ts=2026-05-09T20:12:38.853128753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:12:38.856289344Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:12:39.015450718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:12:41.969038495Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-09T20:12:44.016010708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:12:44.810487971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:12:44.821219341Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:12:45.030137873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:12:45.033496016Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=debug ts=2026-05-09T20:12:45.222044185Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-09T20:12:45.222067687Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-09T20:12:45.947968045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:12:45.951422387Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:12:48.474808253Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T20:12:49.015978411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:12:49.151337534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:12:49.154232668Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T20:12:51.158120902Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T20:12:51.251317664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52526"
level=debug ts=2026-05-09T20:12:51.728583493Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-09T20:12:52.025479431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:12:52.028508141Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:12:53.40551722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:12:53.409018514Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:12:53.40904002Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:12:53.409062183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:12:53.411875212Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:12:53.459956956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:12:53.46278437Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:12:53.462800997Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:12:53.709532883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:12:53.712449126Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:12:53.809835418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:12:53.812908656Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-09T20:12:54.015353718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:12:54.429551545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:12:54.432479094Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:12:55.004063639Z caller=checkpoint.go:372 msg="writing series" size="19 MB"
level=debug ts=2026-05-09T20:12:55.004088504Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=3
ts=2026-05-09T20:12:55.962137489Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:12:58.234571919Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T20:12:58.234598917Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-09T20:12:58.356150091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:12:58.361444479Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=warn ts=2026-05-09T20:12:59.016240998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:12:59.242333501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:12:59.246208346Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T20:13:01.487236141Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-05-09T20:13:01.573411474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:13:01.579845673Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:13:03.080302562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:13:03.083665078Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
level=info ts=2026-05-09T20:13:03.713808253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:13:03.716983388Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:13:03.717002707Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:13:03.717027996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:13:03.719383561Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:13:03.765451027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:13:03.768174914Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:13:03.770253638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:13:03.77252397Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:13:03.99758702Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:13:03.998001183Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:13:04.015261801Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:13:04.015284762Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:13:04.015293032Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:13:04.015293117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:13:04.739793753Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-09T20:13:05.60491217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:13:05.611668147Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:13:07.994286526Z caller=checkpoint.go:372 msg="writing series" size="652 kB"
level=warn ts=2026-05-09T20:13:09.015779036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:13:09.147065758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:13:09.150163829Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:13:09.150183955Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:13:09.150207303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:13:09.152523578Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:13:10.826216327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:13:10.829531497Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:13:11.245967282Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=info ts=2026-05-09T20:13:12.682215998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:13:12.686815326Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T20:13:14.016127885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:13:14.499611058Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T20:13:14.499633092Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
ts=2026-05-09T20:13:14.555741483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:55678"
level=info ts=2026-05-09T20:13:15.049812173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:13:15.056629181Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:13:17.752153617Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T20:13:18.638686453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:13:18.642147904Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T20:13:19.015816039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:13:19.56530236Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:13:19.565298516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="593 kB" avg_comp="593 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" idle=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:13:19.580724842Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=info ts=2026-05-09T20:13:19.901438422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:13:19.904452637Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:13:19.957524857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:13:19.960485099Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:13:21.005727938Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-09T20:13:21.097891592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:13:21.10543708Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-09T20:13:21.252401529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33570"
level=info ts=2026-05-09T20:13:23.651216632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:13:23.655243602Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T20:13:24.016130242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:13:24.10339892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:13:24.107133165Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:13:24.257877334Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
ts=2026-05-09T20:13:25.964813611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:13:27.511471886Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=warn ts=2026-05-09T20:13:29.015634935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:13:30.764015829Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-09T20:13:30.869069849Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T20:13:32.495738117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:13:32.499660967Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=info ts=2026-05-09T20:13:33.341707019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:13:33.34477422Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:13:34.015680692Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:13:34.015710045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:13:34.015718271Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:13:34.015718797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:13:34.016815876Z caller=checkpoint.go:384 msg="flushing series" totalSize="869 kB" series=5
level=info ts=2026-05-09T20:13:34.023672058Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-09T20:13:34.025088746Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00845036s
level=info ts=2026-05-09T20:13:36.373970953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:13:36.377040541Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:13:36.866527164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:13:36.869614768Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:13:37.645873671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:13:37.653401801Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:13:38.716379195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:13:38.719631483Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:13:38.719652982Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:13:38.719674464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:13:38.721999023Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T20:13:39.015852956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:13:39.496951001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:13:39.500406149Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:13:39.500428903Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:13:39.500452486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:13:39.503852913Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=info ts=2026-05-09T20:13:41.875753534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:13:41.879370131Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-09T20:13:43.046894536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:13:43.049941113Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T20:13:44.015731874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:13:44.346727925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:13:44.350444822Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:13:48.350571671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:13:48.356390185Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T20:13:49.01551992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:13:49.598208559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:13:49.604425902Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:13:49.60444542Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:13:49.604467329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:13:49.609755274Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:13:49.663860091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:13:49.666921845Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:13:50.848392708Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:13:51.562303163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:13:51.565409292Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:13:52.908218837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:13:52.9113569Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T20:13:54.015591679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:13:54.337319527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:13:54.341067835Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T20:13:55.088218544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:13:55.092613194Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:13:55.534772299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:13:55.538357339Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
ts=2026-05-09T20:13:55.967643078Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T20:13:58.393706063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:13:58.397067418Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=warn ts=2026-05-09T20:13:59.015499073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:14:00.712300495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:14:00.716529865Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:14:00.716550902Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:14:00.716574536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:14:00.718935631Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:14:02.181987886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:14:02.18984988Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:14:03.843019481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:14:03.848669905Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:14:03.997512051Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:14:03.997757034Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:14:04.016079724Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T20:14:04.016106768Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:14:04.016123765Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:14:04.016133895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:14:04.016145189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:14:04.0162415Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-09T20:14:04.016250793Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T20:14:04.016274164Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.839µs success=true
level=debug ts=2026-05-09T20:14:04.016552945Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T20:14:04.666316639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:14:04.669125128Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T20:14:05.929691149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:14:05.93356925Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:14:07.269997555Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
ts=2026-05-09T20:14:07.450058748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:37076"
level=warn ts=2026-05-09T20:14:09.015222851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:14:10.523022424Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T20:14:12.413081234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:14:12.416881087Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=info ts=2026-05-09T20:14:12.73873225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:14:12.741591087Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:14:13.125572794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:14:13.776449201Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
level=warn ts=2026-05-09T20:14:14.015187007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:14:14.573848362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:14:14.576621947Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:14:17.029026577Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T20:14:18.069958546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:14:18.073989312Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T20:14:19.01600431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:14:19.543729191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:14:19.55014807Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T20:14:20.282084873Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=info ts=2026-05-09T20:14:20.665124653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:14:20.671798353Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:14:20.671820639Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:14:20.67184671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:14:20.678247993Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:14:20.682450381Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T20:14:21.257813795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53546"
level=info ts=2026-05-09T20:14:21.607585288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="592 B" avg_comp="592 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:14:21.610541822Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-09T20:14:23.5351124Z caller=checkpoint.go:372 msg="writing series" size="363 kB"
level=debug ts=2026-05-09T20:14:23.535140569Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-05-09T20:14:23.949458774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:14:23.952690122Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T20:14:24.015764506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:14:25.970270002Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:14:26.78855631Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=info ts=2026-05-09T20:14:28.662479282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:14:28.665487026Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T20:14:29.015504104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:14:30.041615638Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-09T20:14:30.725302091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:14:30.730033088Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:14:31.225750512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:14:31.229560489Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:14:31.229583875Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:14:31.229609107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:14:31.23219179Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:14:31.864963584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:14:31.871654007Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:14:32.255847855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:14:32.258951928Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T20:14:33.293832681Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T20:14:33.346831027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:14:33.350311574Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=info ts=2026-05-09T20:14:33.353341053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:14:33.355865752Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T20:14:34.015221562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:14:34.015225756Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:14:34.015246282Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:14:34.015255221Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:14:34.03245466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:14:34.03868614Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:14:34.756218139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="1.6 MB" avg_comp="818 kB" total_uncomp="13 MB" avg_uncomp="6.4 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:14:34.75960497Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e0e098574%3A19e0e0afd88%3A3cc8e64a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-09T20:14:34.759624927Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:14:34.759647933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:14:34.762692935Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-09T20:14:36.547598435Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-09T20:14:37.453074784Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:54984"
level=warn ts=2026-05-09T20:14:39.015970602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:14:39.497199973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:14:39.500349007Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:14:39.515402316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:14:39.520351175Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:14:39.800780589Z caller=checkpoint.go:372 msg="writing series" size="596 kB"
level=debug ts=2026-05-09T20:14:39.800799941Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-09T20:14:40.725938676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:14:40.729757795Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T20:14:41.327469816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:14:41.330482689Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:14:42.219068885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:14:42.22576794Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:14:43.053037927Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-09T20:14:44.015652159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:14:46.305697125Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-09T20:14:46.997657386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:14:47.000847711Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:14:48.292488016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:14:48.29655362Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
level=warn ts=2026-05-09T20:14:49.0161372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:14:49.162502926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:14:49.166472682Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:14:49.179537287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:14:49.182142171Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:14:49.182155509Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:14:49.182174462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:14:49.184474209Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T20:14:49.362424248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:14:49.36651718Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-09T20:14:49.559497959Z caller=checkpoint.go:372 msg="writing series" size="694 kB"
level=debug ts=2026-05-09T20:14:51.165223234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:14:51.165264249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="176 kB" avg_comp="176 kB" total_uncomp="893 kB" avg_uncomp="893 kB" idle=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:14:51.174665249Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
ts=2026-05-09T20:14:51.259081361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55336"
level=info ts=2026-05-09T20:14:52.144468327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:14:52.148077118Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-09T20:14:52.812361346Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-09T20:14:54.015955055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:14:55.971272898Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:14:56.065577613Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T20:14:59.015560511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:14:59.320581595Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-09T20:14:59.320602885Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=6
level=info ts=2026-05-09T20:14:59.509451976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:14:59.514180261Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=info ts=2026-05-09T20:15:00.327883541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:15:00.335392822Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:15:00.488764297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:15:00.493246991Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:15:02.571711744Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=info ts=2026-05-09T20:15:02.908994518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:15:02.911753895Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T20:15:03.997444773Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:15:03.998480478Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:15:03.999769526Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-09T20:15:03.999815484Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975952 n=1
level=debug ts=2026-05-09T20:15:03.99985501Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975952 n=1
level=info ts=2026-05-09T20:15:04.01584912Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:15:04.015851094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:15:04.015868934Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:15:04.015876827Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T20:15:04.67793025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T20:15:05.603346215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:15:05.606358437Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:15:05.824044869Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-09T20:15:07.817809433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:15:07.820694127Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-09T20:15:09.015980896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:15:09.137512395Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-09T20:15:09.137537705Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=info ts=2026-05-09T20:15:10.31831088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:15:10.32218395Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=debug ts=2026-05-09T20:15:12.33015687Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-09T20:15:12.573793774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:15:12.576853291Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:15:14.015276782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:15:14.062471839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:15:14.065611477Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:15:14.504757758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:15:14.508399628Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:15:14.815133946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:15:14.818940593Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-09T20:15:15.58325097Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T20:15:17.794898568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:15:17.798825178Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:15:18.743321389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:15:18.748145492Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=debug ts=2026-05-09T20:15:18.836217405Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-09T20:15:19.01565919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:15:22.089957611Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=debug ts=2026-05-09T20:15:22.26925705Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:15:23.069226152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:15:23.076674393Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T20:15:24.015383454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:15:24.98084671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:15:24.984342271Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:15:25.343111407Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
ts=2026-05-09T20:15:25.973706254Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:15:28.595252949Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-09T20:15:29.015266321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:15:29.750159998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:15:29.758963148Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:15:29.75898237Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:15:29.759021261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:15:29.766514613Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:15:30.322869055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:15:30.326841133Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:15:30.698960683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:15:30.705137161Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:15:31.848760903Z caller=checkpoint.go:372 msg="writing series" size="321 kB"
level=debug ts=2026-05-09T20:15:31.848783408Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-09T20:15:33.706834417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:15:33.711862278Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:15:34.015789711Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:15:34.015820756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-09T20:15:34.015831132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:15:34.01583806Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T20:15:35.101139806Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-09T20:15:36.820836039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:15:36.824086878Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:15:37.739434221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:15:37.742358541Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:15:37.923943704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:15:37.927579469Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T20:15:38.295765519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:15:38.302720001Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:15:38.354885179Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-09T20:15:39.015822808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:15:39.845296442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:15:39.848061624Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:15:41.607172206Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=info ts=2026-05-09T20:15:42.45566996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:15:42.462909631Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:15:43.465793681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:15:43.473426823Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T20:15:44.01519892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:15:44.325539803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:15:44.328918758Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
ts=2026-05-09T20:15:44.567987907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:46468"
level=debug ts=2026-05-09T20:15:44.862110022Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T20:15:44.862132374Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-09T20:15:44.884755813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:15:44.888563439Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:15:45.685817427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:15:45.689617071Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
level=info ts=2026-05-09T20:15:47.014792075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:15:47.017929019Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-09T20:15:48.113218966Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T20:15:48.311792813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:15:48.31612098Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=warn ts=2026-05-09T20:15:49.016129425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:15:51.263275417Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59240"
level=debug ts=2026-05-09T20:15:51.367085585Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-09T20:15:53.415216736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:15:53.420131615Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:15:53.905632936Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T20:15:54.015945345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:15:54.535567452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:15:54.538965164Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:15:54.619591347Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-09T20:15:55.902755376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:15:55.90605678Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-09T20:15:55.977376298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:15:57.872854261Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=debug ts=2026-05-09T20:15:58.666705381Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T20:15:59.015709715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:15:59.686914045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:15:59.690376027Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:16:01.126043632Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T20:16:03.997586309Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:16:03.998020001Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:16:04.015297109Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:16:04.015320486Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:16:04.015328962Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:16:04.015329274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:16:04.379000266Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=info ts=2026-05-09T20:16:05.231748405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:16:05.237865802Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=debug ts=2026-05-09T20:16:07.631683611Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-09T20:16:08.727998043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:16:08.736020234Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T20:16:09.015704914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:16:10.884996054Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-09T20:16:11.444647283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:16:11.447894585Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:16:12.806930803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:16:12.811051249Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-09T20:16:13.854919166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:16:13.857986826Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T20:16:14.015456628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:16:14.138758278Z caller=checkpoint.go:372 msg="writing series" size="801 kB"
level=debug ts=2026-05-09T20:16:14.138780961Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=9
level=info ts=2026-05-09T20:16:15.045223646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:16:15.048273932Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:16:15.370029997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:16:15.373021293Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:16:17.390273317Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-05-09T20:16:19.015916142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:16:20.653725153Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T20:16:20.653760657Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=info ts=2026-05-09T20:16:22.374356044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:16:22.37812807Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:16:22.37814724Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:16:22.378170494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:16:22.381165833Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:16:22.469497896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:16:22.472727887Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:16:23.897174212Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T20:16:24.015493441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:16:24.254504587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:16:24.273897588Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:16:24.961724199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:16:24.965650285Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T20:16:25.249894075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:16:25.253144041Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-09T20:16:25.980240316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T20:16:26.739739174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:16:26.746962087Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:16:27.150375133Z caller=checkpoint.go:372 msg="writing series" size="488 kB"
level=debug ts=2026-05-09T20:16:27.270680598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:16:28.914288866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:16:28.918813496Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T20:16:29.015870834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:16:29.980822579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:16:29.983774267Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:16:30.402348961Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-09T20:16:33.182142653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:16:33.185352879Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=debug ts=2026-05-09T20:16:33.65663857Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T20:16:33.656659532Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=info ts=2026-05-09T20:16:34.015826783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:16:34.015824854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:16:34.015852541Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:16:34.015862745Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:16:34.050991139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:16:34.054013374Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:16:34.69688387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:16:34.70138032Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=info ts=2026-05-09T20:16:36.146593475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:16:36.149917256Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T20:16:36.90868685Z caller=checkpoint.go:372 msg="writing series" size="664 kB"
ts=2026-05-09T20:16:37.462162059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:47514"
level=info ts=2026-05-09T20:16:37.507868753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=724e2a1e667758b3 immediate=false num_chunks=1 total_comp="404 kB" avg_comp="404 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:16:37.512659738Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/724e2a1e667758b3/19e0e1a368e%3A19e0e254696%3A437451?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-efbl\"}"
level=error ts=2026-05-09T20:16:37.512680098Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:16:37.512701756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" idle=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:16:37.52114136Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=info ts=2026-05-09T20:16:37.801106306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:16:37.807174129Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:16:37.807201058Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:16:37.813367526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:16:37.818860191Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-09T20:16:39.015499986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:16:40.161589262Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-09T20:16:41.409404062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:16:41.413571464Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:16:41.518825181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:16:41.522183126Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:16:43.416946163Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-09T20:16:43.416968698Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=3
level=warn ts=2026-05-09T20:16:44.015634612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:16:44.114924106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:16:44.118192264Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:16:44.11821627Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:16:44.118239498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:16:44.121142687Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:16:44.347943206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:16:44.350776277Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:16:46.667754937Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-09T20:16:49.015266344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:16:49.525591625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:16:49.529559507Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=debug ts=2026-05-09T20:16:49.921061657Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-09T20:16:50.925926649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:16:50.929202883Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:16:53.173240806Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-09T20:16:54.015838655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:16:54.787647186Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:16:54.789317243Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-09T20:16:55.45136761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:16:55.454977538Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-05-09T20:16:55.982525856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:16:56.426756522Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-09T20:16:56.7235622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:16:56.731477993Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:16:57.273395805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:16:57.276672485Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=info ts=2026-05-09T20:16:57.937442997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:16:57.940354799Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T20:16:59.015524566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:16:59.684045144Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-09T20:16:59.684065434Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=5
level=debug ts=2026-05-09T20:17:01.794646238Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:17:02.500397635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:17:02.507316497Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:17:02.855212831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:17:02.857967789Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:17:02.933333677Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=info ts=2026-05-09T20:17:03.972675826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:17:03.976175568Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:17:03.997518329Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:17:03.997703582Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:17:04.016031624Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:17:04.016038395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:17:04.016047598Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:17:04.016056322Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:17:06.07621932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:17:06.079139288Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:17:06.188024157Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-09T20:17:06.188045386Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=2
ts=2026-05-09T20:17:07.465252556Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:33090"
level=info ts=2026-05-09T20:17:07.612067361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:17:07.618233615Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-09T20:17:09.015683181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:17:09.439190508Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-09T20:17:10.654393043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:17:10.660733959Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:17:11.656597894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:17:11.663816027Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:17:12.111001094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:17:12.11520615Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T20:17:12.692135367Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
level=info ts=2026-05-09T20:17:13.246472962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:17:13.249513643Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-09T20:17:14.016065876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:17:14.274078013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:17:14.27780954Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:17:15.017426021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:17:15.020925179Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:17:15.514875361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:17:15.51784197Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T20:17:15.945431593Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
level=info ts=2026-05-09T20:17:16.624143235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:17:16.626960969Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-09T20:17:19.015222799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:17:19.198427354Z caller=checkpoint.go:372 msg="writing series" size="656 kB"
level=debug ts=2026-05-09T20:17:19.198446445Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-05-09T20:17:19.970974057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:17:19.974630222Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
ts=2026-05-09T20:17:21.271545842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38982"
level=info ts=2026-05-09T20:17:22.035503459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:17:22.038643138Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=info ts=2026-05-09T20:17:22.277156012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:17:22.284214891Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:17:22.45099004Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-09T20:17:23.079716106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:17:23.082876384Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:17:23.434010733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:17:23.438662736Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:17:23.438685534Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:17:23.438708626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:17:23.441886761Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T20:17:24.015563482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:17:25.208436482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:17:25.216751171Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:17:25.704017988Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-09T20:17:25.945810861Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T20:17:25.98415491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=info ts=2026-05-09T20:17:27.713888342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:17:27.717107753Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:17:28.573435584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:17:28.578028699Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=debug ts=2026-05-09T20:17:28.957626257Z caller=checkpoint.go:372 msg="writing series" size="668 kB"
level=warn ts=2026-05-09T20:17:29.015210566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:17:32.210163979Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T20:17:34.015998207Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:17:34.016010513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:17:34.016024488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:17:34.016032721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T20:17:35.462561042Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-09T20:17:37.468734793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:40836"
level=info ts=2026-05-09T20:17:38.102145658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:17:38.106648265Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-09T20:17:38.715594127Z caller=checkpoint.go:372 msg="writing series" size="356 kB"
level=debug ts=2026-05-09T20:17:38.715619155Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-09T20:17:38.921912967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:17:38.924980171Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:17:39.015833285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:17:40.661395457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:17:40.664972718Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:17:41.099776636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:17:41.968956893Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T20:17:42.533824109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:17:42.537247167Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T20:17:44.015753914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:17:44.577389528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:58812"
level=debug ts=2026-05-09T20:17:45.222002284Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-09T20:17:46.80155313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:17:46.804528339Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:17:48.475165264Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=warn ts=2026-05-09T20:17:49.015596232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:17:49.806032853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:17:49.809292977Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:17:51.242516407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:17:51.247928403Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:17:51.728456002Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-09T20:17:52.307141035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:17:52.310752396Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:17:52.310769952Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:17:52.310790982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:17:52.313617289Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-09T20:17:54.015350042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:17:54.964141861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:17:54.967038141Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:17:54.967056476Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:17:54.96707665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:17:54.969383552Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:17:55.001705147Z caller=checkpoint.go:372 msg="writing series" size="20 MB"
level=debug ts=2026-05-09T20:17:55.001730022Z caller=checkpoint.go:384 msg="flushing series" totalSize="20 MB" series=5
level=info ts=2026-05-09T20:17:55.006448458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:17:55.009785644Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
ts=2026-05-09T20:17:55.986801923Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T20:17:58.234254894Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-09T20:17:58.234279478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=warn ts=2026-05-09T20:17:59.015846923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:18:00.448666694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:18:00.454515508Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:18:00.454540515Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:18:00.454562173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:18:00.456947892Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:18:01.023298319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:18:01.029922201Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:18:01.487208495Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=debug ts=2026-05-09T20:18:01.685879989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:18:01.833257267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:18:01.837321557Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:18:01.837341035Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:18:01.83736259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:18:01.840797458Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:18:03.255052877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:18:03.258860559Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:18:03.836551799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:18:03.83959434Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:18:03.997556217Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:18:03.998022814Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:18:04.015287062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:18:04.015294606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:18:04.015314646Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:18:04.01532495Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:18:04.232754069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:18:04.236632175Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:18:04.740448332Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-09T20:18:06.911933493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:18:06.915128196Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T20:18:07.397889216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:18:07.401523132Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T20:18:07.993454212Z caller=checkpoint.go:372 msg="writing series" size="692 kB"
level=info ts=2026-05-09T20:18:08.740975423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:18:08.753300458Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T20:18:09.015975974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:18:10.298133256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:18:10.30200611Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=info ts=2026-05-09T20:18:10.918780487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:18:10.921456102Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:18:11.245701318Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-09T20:18:11.269673313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:18:11.276826603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:18:13.451041466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:18:13.453999348Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-09T20:18:14.015422947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:18:14.28454078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:18:14.287381713Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:18:14.311488713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:18:14.314187753Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:18:14.499149545Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T20:18:14.499172249Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
ts=2026-05-09T20:18:14.580474069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:52928"
level=info ts=2026-05-09T20:18:15.879715408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:18:15.883489379Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=info ts=2026-05-09T20:18:16.590886218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:18:16.593900022Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:18:17.223769951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:18:17.227244221Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=debug ts=2026-05-09T20:18:17.751635207Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=warn ts=2026-05-09T20:18:19.016198607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:18:20.353745513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:18:20.357065039Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:18:21.005327389Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
ts=2026-05-09T20:18:21.276444533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37450"
level=info ts=2026-05-09T20:18:21.358081291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:18:21.36230119Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T20:18:24.016082598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:18:24.25817274Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-09T20:18:24.274281782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:18:24.277997678Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-09T20:18:25.990042148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:18:27.511685448Z caller=checkpoint.go:372 msg="writing series" size="673 kB"
level=info ts=2026-05-09T20:18:28.053373226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:18:28.056702091Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-09T20:18:29.01586382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:18:29.433208142Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:18:30.763553122Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-09T20:18:30.945082967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:18:30.949657222Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-09T20:18:32.71923532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:18:32.722219001Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:18:32.805756549Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:18:34.016039481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:18:34.016040737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:18:34.01606267Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:18:34.016070423Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-09T20:18:34.017120948Z caller=checkpoint.go:384 msg="flushing series" totalSize="878 kB" series=5
level=info ts=2026-05-09T20:18:34.024188944Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000018.tmp new=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-09T20:18:34.025630683Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009214959s
level=info ts=2026-05-09T20:18:35.741962858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:18:35.745294187Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:18:35.745314864Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:18:35.745337605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:18:35.747778349Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:18:39.015746156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:18:40.15985145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:18:40.163862064Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T20:18:40.812776499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:18:40.816156333Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=warn ts=2026-05-09T20:18:44.015940045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:18:44.231009198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:18:44.234360175Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-09T20:18:44.58361648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:44120"
level=info ts=2026-05-09T20:18:46.078530831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:18:46.081836893Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:18:46.22438125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:18:46.227961604Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=info ts=2026-05-09T20:18:46.507840108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:18:46.513914404Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T20:18:49.015543007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:18:49.29773434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:18:49.30513427Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:18:51.549610164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:18:51.55664255Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:18:53.058808288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:18:53.06247656Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T20:18:54.015296217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:18:54.500604785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:18:54.503705815Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=info ts=2026-05-09T20:18:55.223109369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:18:55.22606498Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-09T20:18:55.993028546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T20:18:56.747565663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:18:56.751557257Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=warn ts=2026-05-09T20:18:59.0161403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:19:00.105750111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:19:00.112334448Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:19:01.461680909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:19:01.466078341Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=debug ts=2026-05-09T20:19:03.997411822Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:19:03.998457842Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T20:19:04.0158189Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-09T20:19:04.015825935Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:19:04.015831124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:19:04.015844008Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:19:04.015854081Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:19:04.015959038Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000019
level=info ts=2026-05-09T20:19:04.015966505Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-09T20:19:04.016000441Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.128µs success=true
level=debug ts=2026-05-09T20:19:04.016363413Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-09T20:19:04.636555258Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:19:04.720716236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:19:04.724322985Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:19:05.659867344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:19:05.667172411Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:19:06.480483385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:19:06.485177469Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=debug ts=2026-05-09T20:19:07.269581281Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-09T20:19:09.015659824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:19:09.248427996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:19:09.251400642Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:19:09.398912065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:19:09.401781754Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:19:09.71430677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:19:09.718030212Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:19:09.778818162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:19:09.781794489Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:19:10.522920757Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-09T20:19:13.776315121Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
level=warn ts=2026-05-09T20:19:14.015924487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:19:15.007874323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:19:15.011195Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:19:15.824568766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T20:19:15.914777574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:19:15.917712316Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:19:17.029181955Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-09T20:19:17.195776378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:19:17.199030428Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T20:19:17.271129795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:19:17.274212965Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:19:19.01577715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:19:20.282697687Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
level=info ts=2026-05-09T20:19:22.350336133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:19:22.354723387Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T20:19:23.535332643Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-09T20:19:23.535354182Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-09T20:19:24.01548424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:19:25.996358813Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T20:19:26.722208625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:19:26.726408255Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-09T20:19:26.787630353Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=info ts=2026-05-09T20:19:28.260921995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:19:28.266518036Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T20:19:29.016159564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:19:30.041134758Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-09T20:19:31.631341721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:19:31.636076252Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:19:31.63611041Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:19:31.636132933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:19:31.640758116Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T20:19:33.293656857Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-09T20:19:33.810276264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="592 B" avg_comp="592 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:19:33.813265226Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:19:34.015927491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:19:34.015956567Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:19:34.015965969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:19:34.016006034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:19:35.459180171Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:19:35.459233953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="476 kB" avg_comp="476 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T20:19:35.462558015Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0e09362a%3A19e0e629de3%3A86fdf17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T20:19:36.308150818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:19:36.314592595Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:19:36.547176348Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-09T20:19:37.597232365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:19:37.600335706Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:19:37.600354617Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:19:37.600381709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:19:37.60279368Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:19:37.84846223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:19:37.851811152Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=warn ts=2026-05-09T20:19:39.015901649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:19:39.629158659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:19:39.636437335Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:19:39.799781993Z caller=checkpoint.go:372 msg="writing series" size="596 kB"
level=debug ts=2026-05-09T20:19:39.799801437Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-09T20:19:40.099143106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:19:40.102486394Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=debug ts=2026-05-09T20:19:43.053193851Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-09T20:19:43.752147748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:19:43.75582513Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T20:19:44.015511405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:19:44.078737264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:19:44.081739487Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
ts=2026-05-09T20:19:44.589399503Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:39720"
level=info ts=2026-05-09T20:19:45.91756282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:19:45.921253919Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=info ts=2026-05-09T20:19:46.010790768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:19:46.013490318Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:19:46.013509481Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:19:46.013528391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:19:46.015789823Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:19:46.147027762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:19:46.149872651Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=debug ts=2026-05-09T20:19:46.306461065Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-09T20:19:49.016033761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:19:49.151410539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:19:49.155182965Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:19:49.155203051Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:19:49.155224737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:19:49.158639856Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=info ts=2026-05-09T20:19:49.224180215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="478 kB" avg_comp="478 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T20:19:49.227400236Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0e09362a%3A19e0e629de3%3A86fdf17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-09T20:19:49.356673623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:19:49.359489898Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-09T20:19:49.559176569Z caller=checkpoint.go:372 msg="writing series" size="778 kB"
level=debug ts=2026-05-09T20:19:52.812108295Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-09T20:19:53.83806291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:19:53.842393438Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-09T20:19:54.016021871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-09T20:19:55.999785705Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=debug ts=2026-05-09T20:19:56.065164318Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-09T20:19:57.187352119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:19:57.193608164Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T20:19:59.015978135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:19:59.31996539Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-09T20:19:59.319987441Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.6 MB" series=6
level=debug ts=2026-05-09T20:19:59.621388387Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:20:00.552436505Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:20:01.038108303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:20:01.041497102Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:20:01.996203636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:20:02.000181715Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:20:02.000200635Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:20:02.000221508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:20:02.002601655Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-09T20:20:02.571164878Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=info ts=2026-05-09T20:20:03.036694494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:20:03.039481354Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:20:03.039501711Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:20:03.03952316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:20:03.041698076Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:20:03.997223505Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:20:03.998273837Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T20:20:04.015654117Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:20:04.015656808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:20:04.015680224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:20:04.015688116Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:20:04.531387868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:20:04.534352618Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:20:05.153146867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:20:05.156379666Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:20:05.15640026Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:20:05.156422196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:20:05.15885473Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-09T20:20:05.598206569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:20:05.601820886Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T20:20:05.824416666Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-09T20:20:06.05114536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:20:06.054006536Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:20:07.995435057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:20:07.999273226Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=info ts=2026-05-09T20:20:08.289064632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:20:08.291911484Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-09T20:20:09.015464856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:20:09.131713479Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-09T20:20:09.131742194Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=debug ts=2026-05-09T20:20:12.330332354Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-09T20:20:12.622236141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:20:12.626375844Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-09T20:20:14.015511868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:20:14.188963836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:20:14.192086664Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=info ts=2026-05-09T20:20:14.579136897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="478 kB" avg_comp="478 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T20:20:14.583219273Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0e09362a%3A19e0e629de3%3A86fdf17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-09T20:20:14.592885763Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.89:53456"
level=info ts=2026-05-09T20:20:15.31074608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:20:15.313977493Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:20:15.582811388Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-09T20:20:16.389067831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:20:16.393580996Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:20:16.470341167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:20:16.477568318Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:20:17.364014131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:20:17.367066576Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:20:18.182411547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:20:18.185596384Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:20:18.836205009Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-09T20:20:19.01578773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:20:19.815551709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:20:19.818787532Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:20:20.074240061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:20:20.081466588Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:20:20.081490101Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:20:20.081513353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:20:20.08685812Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:20:20.989553161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:20:20.996940796Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-09T20:20:21.287742122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47736"
level=debug ts=2026-05-09T20:20:22.089791976Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
level=info ts=2026-05-09T20:20:23.81523546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:20:23.818251014Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-09T20:20:24.015851241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:20:24.065964025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:20:24.070040488Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=debug ts=2026-05-09T20:20:25.342521185Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=info ts=2026-05-09T20:20:25.48546876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:20:25.488446474Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-09T20:20:26.003238978Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:20:26.72480719Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:20:27.154977164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:20:27.158838705Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-09T20:20:28.595263339Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-09T20:20:29.016108497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:20:29.901719927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:20:29.905230504Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=info ts=2026-05-09T20:20:30.630451453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:20:30.63394112Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:20:30.633959072Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:20:30.633979966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:20:30.636819288Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=info ts=2026-05-09T20:20:31.015150318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:20:31.019324354Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=debug ts=2026-05-09T20:20:31.848757001Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=info ts=2026-05-09T20:20:33.541573564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:20:33.547807752Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=info ts=2026-05-09T20:20:34.015180628Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:20:34.015208155Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:20:34.015218351Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:20:34.015232671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:20:35.10168419Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=debug ts=2026-05-09T20:20:37.449334746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-09T20:20:37.450815004Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-09T20:20:37.480405051Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:49362"
level=info ts=2026-05-09T20:20:38.337499743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:20:38.340637235Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:20:38.353759315Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-09T20:20:38.578460915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:20:38.584254274Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-09T20:20:39.015277918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:20:41.390706937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:20:41.395294635Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T20:20:41.607222275Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=debug ts=2026-05-09T20:20:41.607240192Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-09T20:20:44.015846388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:20:44.293197946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:20:44.296256148Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:20:44.861305251Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-09T20:20:44.86132796Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-09T20:20:46.543142542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:20:46.546230683Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:20:46.836214636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:20:46.839247869Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-09T20:20:48.113106001Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-09T20:20:49.015910415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:20:50.360522504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:20:50.363819583Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T20:20:50.40510906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:20:50.408370594Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
ts=2026-05-09T20:20:51.288788301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:58530"
level=debug ts=2026-05-09T20:20:51.366498621Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-09T20:20:54.016168776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:20:54.618949423Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=debug ts=2026-05-09T20:20:55.323459918Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-09T20:20:56.005772357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-tpcdf-6edfcc94 10.130.0.91:7946"
level=info ts=2026-05-09T20:20:56.731373148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:20:56.738657433Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:20:56.73867726Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:20:56.738701071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:20:56.745046842Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-09T20:20:57.523298627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="478 kB" avg_comp="478 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T20:20:57.526994051Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0e09362a%3A19e0e629de3%3A86fdf17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-09T20:20:57.872167714Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=info ts=2026-05-09T20:20:58.790655864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:20:58.794754578Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T20:20:59.015658436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:20:59.241132252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:20:59.24419138Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:20:59.244213272Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:20:59.244239356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:20:59.247269991Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:20:59.878409974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:20:59.881462732Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:20:59.881483872Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:20:59.881505013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:20:59.883751727Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-09T20:21:01.125592969Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-09T20:21:01.834663653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:21:01.843697152Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:21:03.430215897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:21:03.433771966Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T20:21:03.997444629Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:21:03.998489253Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T20:21:04.015901377Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:21:04.015926476Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:21:04.015936194Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:21:04.015985166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:21:04.37796153Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-09T20:21:05.257035192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:21:05.262222929Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:21:05.262240084Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:21:05.262261468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:21:05.265354946Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-09T20:21:07.631179727Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-09T20:21:08.073826714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:21:08.07925065Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:21:08.410222087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:21:08.41305577Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:21:08.630676373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:21:08.63369882Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:21:09.015572256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:21:10.884696357Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-09T20:21:11.534389342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:21:11.540578485Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:21:12.422334777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:21:12.425235636Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:21:13.373752538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:21:13.376670433Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:21:14.005557392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:21:14.009111353Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T20:21:14.015292871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:21:14.138400435Z caller=checkpoint.go:372 msg="writing series" size="817 kB"
level=debug ts=2026-05-09T20:21:14.138422886Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=9
level=info ts=2026-05-09T20:21:14.155767522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:21:14.162149427Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:21:17.390011908Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-09T20:21:17.46420787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:21:17.467361152Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T20:21:19.01253089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-09T20:21:19.01517274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-09T20:21:19.015534657Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:21:19.937357983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:21:19.941233105Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-09T20:21:20.655816684Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-09T20:21:20.655839633Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=info ts=2026-05-09T20:21:21.07682526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:21:21.080496563Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=debug ts=2026-05-09T20:21:23.896660386Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=warn ts=2026-05-09T20:21:24.015676023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:21:24.050802873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:21:24.054357769Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=info ts=2026-05-09T20:21:25.149226741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:21:25.15325254Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
ts=2026-05-09T20:21:26.007786111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-d7cfd763 10.128.0.74:7946"
level=debug ts=2026-05-09T20:21:27.149930315Z caller=checkpoint.go:372 msg="writing series" size="504 kB"
level=info ts=2026-05-09T20:21:27.739286834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:21:27.742639343Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:21:27.742660238Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:21:27.742684543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:21:27.745354103Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=info ts=2026-05-09T20:21:28.686933107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:21:28.690689879Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=warn ts=2026-05-09T20:21:29.015987553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:21:30.402787252Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=debug ts=2026-05-09T20:21:30.774599971Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:21:33.656953164Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-09T20:21:33.656975933Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=info ts=2026-05-09T20:21:34.015771682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-09T20:21:34.015784598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:21:34.015796242Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:21:34.015805458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-09T20:21:35.644418078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:21:35.647230744Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:21:36.013561622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:21:36.017635463Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T20:21:36.909012813Z caller=checkpoint.go:372 msg="writing series" size="567 kB"
level=debug ts=2026-05-09T20:21:37.277921145Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-09T20:21:37.782218448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:21:37.785074453Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-09T20:21:39.015681661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:21:39.030824764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:21:39.034589378Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-09T20:21:39.470328879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:21:39.473521352Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:21:40.161811967Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-09T20:21:40.445653639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:21:40.448799908Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:21:41.073709509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:21:41.07682559Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-09T20:21:42.089717393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:21:42.095977963Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:21:43.417017882Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-09T20:21:43.417039787Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=3
level=info ts=2026-05-09T20:21:43.746393687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:21:43.749773008Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:21:43.82808438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:21:43.831153312Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-09T20:21:44.015480804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:21:45.34838942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:21:45.351827683Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=debug ts=2026-05-09T20:21:46.667955068Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-09T20:21:49.015584687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:21:49.920479905Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-05-09T20:21:50.139150631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:21:50.142992833Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-09T20:21:51.297431435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:21:51.301011908Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-09T20:21:51.561678814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:21:51.564647632Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:21:53.082813086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:21:53.089585838Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:21:53.173738747Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-09T20:21:54.016075129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:21:55.41240686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:21:55.418498133Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:21:55.422539215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="478 kB" avg_comp="478 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T20:21:55.4256665Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0e09362a%3A19e0e629de3%3A86fdf17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-09T20:21:56.010495431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:21:56.426318348Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-09T20:21:56.489524589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:21:56.492739595Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-09T20:21:57.210539711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:21:57.21364292Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-05-09T20:21:59.015628813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:21:59.682593471Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-05-09T20:21:59.682614829Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=5
level=debug ts=2026-05-09T20:21:59.834909809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-09T20:22:01.036315752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:22:01.044379361Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:22:01.04440279Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:22:01.044456431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:22:01.051537001Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-09T20:22:01.657141546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:22:01.660646193Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=debug ts=2026-05-09T20:22:02.932441848Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-05-09T20:22:03.997533382Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-09T20:22:03.997899734Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:22:04.015205775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:22:04.015229475Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:22:04.015236873Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:22:04.015248513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:22:04.050593264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:22:04.05521351Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-09T20:22:04.856086121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:22:04.859363417Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=info ts=2026-05-09T20:22:04.925558479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:22:04.928437326Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=debug ts=2026-05-09T20:22:06.188767983Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-09T20:22:06.188792058Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=2
level=warn ts=2026-05-09T20:22:09.015681704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:22:09.438839018Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-05-09T20:22:11.113972455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:22:11.120460205Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=debug ts=2026-05-09T20:22:12.692563896Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
level=info ts=2026-05-09T20:22:13.617918493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:22:13.6214695Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=warn ts=2026-05-09T20:22:14.015833036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:22:15.651229536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-09T20:22:15.944208045Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-09T20:22:16.069717925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:22:16.073611094Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=info ts=2026-05-09T20:22:18.808748638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:22:18.816365262Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-09T20:22:19.015876848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:22:19.198328029Z caller=checkpoint.go:372 msg="writing series" size="656 kB"
level=debug ts=2026-05-09T20:22:19.198347689Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-09T20:22:21.368349568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:22:21.371565442Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-09T20:22:22.450746756Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-05-09T20:22:22.764406763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:22:22.768308279Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T20:22:24.015550341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:22:24.166907617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:22:24.174367191Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-09T20:22:25.703611468Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-05-09T20:22:26.01399883Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=debug ts=2026-05-09T20:22:27.957588256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:22:28.957028445Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=warn ts=2026-05-09T20:22:29.015855806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:22:30.05256191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:22:30.055514823Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:22:30.272259313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:22:30.276035043Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-09T20:22:32.209544991Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-09T20:22:33.236357175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-09T20:22:33.239518759Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e0e09a014%3A19e0e09a015%3A50c07bb1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-09T20:22:33.781846063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-09T20:22:33.78482046Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e0e09bf24%3A19e0e09bf56%3A219b526a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-09T20:22:34.015708832Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:22:34.01573391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:22:34.01574124Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:22:34.01573267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:22:34.050885758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:22:34.055077101Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-09T20:22:34.993509213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:22:34.997071953Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:22:35.462976625Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-09T20:22:36.574010269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:22:36.580139152Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-09T20:22:36.652462479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-09T20:22:36.65521696Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e0e098575%3A19e0e23b905%3Aa5b95deb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-09T20:22:38.638213799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="164 kB" avg_uncomp="164 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-09T20:22:38.641566391Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e0e09a014%3A19e0e09a2c9%3A6bcbf2cb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-09T20:22:38.716264333Z caller=checkpoint.go:372 msg="writing series" size="427 kB"
level=debug ts=2026-05-09T20:22:38.7162849Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-09T20:22:39.016193077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:22:39.932485565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-09T20:22:39.936368952Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e0e0933bf%3A19e0e5bf2db%3A3d5e2a7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-09T20:22:41.96853542Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-09T20:22:43.769466278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="478 kB" avg_comp="478 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-09T20:22:43.775746987Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e0e09362a%3A19e0e629de3%3A86fdf17c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-09T20:22:44.015884644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:22:44.308153728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:22:44.315254646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:22:44.315275911Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:22:44.315305342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:22:44.320770347Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-09T20:22:45.22215982Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-05-09T20:22:45.723533727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="470 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-09T20:22:45.730671428Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e0e093371%3A19e0e18f4e8%3Ad609d7dc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-09T20:22:48.47446674Z caller=checkpoint.go:372 msg="writing series" size="842 B"
level=info ts=2026-05-09T20:22:48.917550227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="511 kB" avg_comp="511 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-09T20:22:48.921877858Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e0e093967%3A19e0e563bba%3A2cf1e80?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-09T20:22:49.015894497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:22:50.595055971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="610 B" avg_comp="610 B" total_uncomp="1.4 kB" avg_uncomp="1.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-09T20:22:50.597953133Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e0e098575%3A19e0e098576%3A6460ac92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-09T20:22:50.856662274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-09T20:22:50.85952057Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e0e098580%3A19e0e09eadb%3A22d0abb7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-09T20:22:51.299919447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39432"
level=debug ts=2026-05-09T20:22:51.727949176Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-09T20:22:53.374427318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="24 kB" avg_comp="24 kB" total_uncomp="252 kB" avg_uncomp="252 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-05-09T20:22:53.377555418Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feadc5651def38e4/19e0e0936e6%3A19e0e09a29f%3A6de21b10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-05-09T20:22:53.807540733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ccc58627b2d40aa immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-phcf\"}"
level=error ts=2026-05-09T20:22:53.810924396Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ccc58627b2d40aa/19e0e19639d%3A19e0e1b5f9b%3A826834e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-phcf\"}"
level=warn ts=2026-05-09T20:22:54.015526819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:22:55.003439462Z caller=checkpoint.go:372 msg="writing series" size="21 MB"
level=debug ts=2026-05-09T20:22:55.003462939Z caller=checkpoint.go:384 msg="flushing series" totalSize="21 MB" series=5
ts=2026-05-09T20:22:56.017340496Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-vlz5d-782c09f8 10.130.0.89:7946"
level=info ts=2026-05-09T20:22:56.961218613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:22:56.964703241Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:22:56.96472425Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:22:56.964746832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:22:56.967695672Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=debug ts=2026-05-09T20:22:58.234481226Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-09T20:22:58.234504169Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=1
level=debug ts=2026-05-09T20:22:58.828066493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-09T20:22:58.835046896Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-09T20:22:59.015569416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:23:01.487015099Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=info ts=2026-05-09T20:23:01.979433329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=19cbbe19d2474278 immediate=false num_chunks=1 total_comp="199 kB" avg_comp="199 kB" total_uncomp="892 kB" avg_uncomp="892 kB" full=1 labels="{stream=\"stat-rep-dlxe\"}"
level=error ts=2026-05-09T20:23:01.98276141Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/19cbbe19d2474278/19e0e1b097d%3A19e0e2250f3%3A607808ab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"stat-rep-dlxe\"}"
level=info ts=2026-05-09T20:23:02.323944046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="161 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-09T20:23:02.331621163Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e0e093b6f%3A19e0e09a29f%3A1842e58d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-09T20:23:02.948327091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ab64abbb21af1223 immediate=false num_chunks=1 total_comp="203 kB" avg_comp="203 kB" total_uncomp="893 kB" avg_uncomp="893 kB" full=1 labels="{stream=\"integration2-snvl\"}"
level=error ts=2026-05-09T20:23:02.952036306Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ab64abbb21af1223/19e0e2594e2%3A19e0e29a7af%3A441846c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-snvl\"}"
level=info ts=2026-05-09T20:23:03.083513186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=941948cc1c9454fd immediate=false num_chunks=1 total_comp="594 kB" avg_comp="594 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-mgvh\"}"
level=error ts=2026-05-09T20:23:03.087006106Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/941948cc1c9454fd/19e0e19ffdd%3A19e0e281351%3A8036252c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mgvh\"}"
level=debug ts=2026-05-09T20:23:03.99724971Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.75:11211 count=1
level=info ts=2026-05-09T20:23:03.998313259Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-09T20:23:04.015664844Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-09T20:23:04.015689739Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-09T20:23:04.015698895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-09T20:23:04.0157353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:23:04.445488409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:23:04.448743766Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:23:04.448761934Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:23:04.44878347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:23:04.451433367Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=debug ts=2026-05-09T20:23:04.739912544Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
ts=2026-05-09T20:23:07.48928126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.91:59446"
level=debug ts=2026-05-09T20:23:07.993442982Z caller=checkpoint.go:372 msg="writing series" size="700 kB"
level=info ts=2026-05-09T20:23:08.016193664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.8 MB" avg_comp="1.9 MB" total_uncomp="26 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-09T20:23:08.0249974Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e0e0933ff%3A19e0e29dba5%3Addd791b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-09T20:23:09.015813854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:23:11.24543443Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-09T20:23:11.837744088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=974da1cb98d9a107 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-mfky\"}"
level=error ts=2026-05-09T20:23:11.841549161Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/974da1cb98d9a107/19e0e18e0b0%3A19e0e255638%3A644c960?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-mfky\"}"
level=warn ts=2026-05-09T20:23:14.015843914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:23:14.49890004Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-09T20:23:14.498922247Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-09T20:23:16.449737177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-09T20:23:16.454787105Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e0e098580%3A19e0e09bf56%3A87609b46?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-09T20:23:17.584932568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-09T20:23:17.587861974Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e0e098580%3A19e0e09bf56%3Aa722dc6d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-09T20:23:17.740288235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.9 MB" avg_comp="1.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-09T20:23:17.746438192Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e0e093778%3A19e0e35b2e7%3A98d26fba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-09T20:23:17.75149633Z caller=checkpoint.go:372 msg="writing series" size="658 B"
level=info ts=2026-05-09T20:23:18.651085486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:23:18.655543557Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:23:18.655568571Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-09T20:23:18.655597754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="414 kB" avg_comp="414 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-09T20:23:18.658906908Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e0e093863%3A19e0e27a20a%3A6902418c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-09T20:23:19.01587936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-09T20:23:20.028154555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-09T20:23:20.031478677Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e0e18decd%3A19e0e1a0f7e%3A50d7682d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-09T20:23:21.004439631Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
ts=2026-05-09T20:23:21.301448142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33684"
level=info ts=2026-05-09T20:23:21.605623878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=267a9260660be6f3 immediate=false num_chunks=1 total_comp="402 kB" avg_comp="402 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"gitlab-rep-gctu\"}"
level=error ts=2026-05-09T20:23:21.609427Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/267a9260660be6f3/19e0e18e880%3A19e0e24dd20%3Ad2a4f506?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-gctu\"}"
level=info ts=2026-05-09T20:23:21.951222844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="739 kB" avg_comp="739 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-09T20:23:21.955607864Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e0e093376%3A19e0e55a17e%3A54c5f958?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-09T20:23:23.142138558Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-09T20:23:24.015917562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-09T20:23:24.257929019Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=info ts=2026-05-09T20:23:24.758280268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="493 kB" avg_comp="493 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-09T20:23:24.762790501Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e0e093789%3A19e0e5c40ce%3Af52ff902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"