level=info ts=2026-05-05T11:25:17.13212228Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-05T11:25:17.132181207Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-05T11:25:17.132224121Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-05T11:25:17.134499629Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-05T11:25:17.134953113Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T11:25:17.135037347Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.90 inf=eth0
level=info ts=2026-05-05T11:25:17.136190915Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-2d81bcce
ts=2026-05-05T11:25:17.136303782Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-05T11:25:17.136326926Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.90 advertisePort=7946
level=debug ts=2026-05-05T11:25:17.136360946Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.90 advertisePort=7946
level=info ts=2026-05-05T11:25:17.136552424Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-05T11:25:17.142071873Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T11:25:17.142113051Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-05T11:25:17.143269538Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-05T11:25:17.143635183Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-05T11:25:17.143652897Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-05T11:25:17.143650835Z
level=debug ts=2026-05-05T11:25:17.143665528Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-05T11:25:17.143690522Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-05T11:25:17.143708743Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-05T11:25:17.143715269Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-05T11:25:17.14371437Z
level=debug ts=2026-05-05T11:25:17.143719699Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-05T11:25:17.14384613Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-05-05T11:25:17.144449501Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T11:25:17.144463142Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-05T11:25:17.144485589Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.936718ms
level=error ts=2026-05-05T11:25:17.144495612Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-05T11:25:17.144503608Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-05-05T11:25:17.146483236Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-05T11:25:17.146573783Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.90 inf=eth0
ts=2026-05-05T11:25:17.150276814Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:17.150292527Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T11:25:17.162536809Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-05-05T11:25:17.162565635Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=info ts=2026-05-05T11:25:17.162576038Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-05T11:25:17.162586653Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-05T11:25:17.162604958Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-05T11:25:17.162613011Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-05-05T11:25:17.162629384Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-05T11:25:17.162667941Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-05T11:25:17.162666144Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-05T11:25:17.162894711Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-05-05T11:25:17.162903955Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-05T11:25:17.162907353Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-05-05T11:25:17.162914248Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-05-05T11:25:17.162938059Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-05-05T11:25:17.162947571Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-05-05T11:25:17.16295964Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-05T11:25:17.162997205Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-05T11:25:17.163003511Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-05-05T11:25:17.163008542Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-05-05T11:25:17.163018809Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-05T11:25:17.163080049Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-05T11:25:17.163092678Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-05T11:25:17.163096381Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-05T11:25:17.163099559Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-05-05T11:25:17.163102315Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-05T11:25:17.163125687Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-05T11:25:17.163134486Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T11:25:17.163195135Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=36.389µs success=true
level=info ts=2026-05-05T11:25:17.16321562Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-05T11:25:17.163306791Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=183.316µs errors=false
level=info ts=2026-05-05T11:25:17.163320165Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-05T11:25:17.163498561Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=375.874µs errors=false
level=info ts=2026-05-05T11:25:17.16350917Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-05T11:25:17.163517428Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=394.298µs
level=info ts=2026-05-05T11:25:17.163936799Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-05T11:25:17.16396704Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-05T11:25:17.16396561Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=2.637861807s
level=info ts=2026-05-05T11:25:17.163994684Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-05T11:25:17.164019095Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-05T11:25:17.164035404Z caller=loki.go:615 msg="Loki started" startup_time=99.521104ms
level=debug ts=2026-05-05T11:25:17.164158243Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-05T11:25:17.164166867Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-05T11:25:17.164248632Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-05T11:25:18.611442875Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:18.611469189Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:25:21.991271599Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:21.991294787Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-05T11:25:22.164138275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:25:27.164852313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:25:29.587852249Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:25:29.587878689Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-05T11:25:32.164633437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:25:37.164962434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:25:40.807096955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.63:7946"
ts=2026-05-05T11:25:40.808789603Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.64:7946: dial tcp 10.130.0.64:7946: connect: connection refused"
level=info ts=2026-05-05T11:25:40.80880636Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=23.664297659s
level=debug ts=2026-05-05T11:25:41.850859834Z caller=logging.go:137 msg="GET /ready (503) 100.305µs"
level=warn ts=2026-05-05T11:25:42.16511174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:25:47.164902798Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-05T11:25:47.164926942Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-05T11:25:47.164933208Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:25:47.164952156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:25:48.849881952Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:25:51.849348547Z caller=logging.go:137 msg="GET /ready (503) 88.683µs"
level=warn ts=2026-05-05T11:25:52.16456955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:25:53.586931051Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:25:57.16417297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:25:58.870810238Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36004"
level=warn ts=2026-05-05T11:26:02.164830348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:26:07.164100451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:26:12.164336502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:26:15.023409734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-05T11:26:17.143706579Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:26:17.144761122Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T11:26:17.164086561Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T11:26:17.164092225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:26:17.164115534Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:26:17.164126259Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T11:26:18.852586594Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:26:20.072011458Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:26:20.072074625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:26:20.081091832Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:26:22.164848065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:26:25.929434975Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:47960"
level=warn ts=2026-05-05T11:26:27.16413491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:26:28.87404109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44934"
level=warn ts=2026-05-05T11:26:32.164591627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:26:32.424287041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T11:26:35.480816772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:26:35.488316919Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:26:37.164923624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:26:42.165086016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:26:47.164645178Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:26:47.164674626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:26:47.164682061Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:26:47.164715101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:26:47.986276263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51412"
ts=2026-05-05T11:26:48.855269323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=warn ts=2026-05-05T11:26:52.16459028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:26:53.872984064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-05T11:26:54.577226103Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:26:57.164820673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:27:02.164327932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:27:04.582484308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:27:04.58968756Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:27:07.164625464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:27:12.16448016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:27:17.143967854Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:27:17.143996162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=warn ts=2026-05-05T11:27:17.164329827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:27:17.164355224Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:27:17.164374679Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:27:17.164382095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T11:27:17.987414404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:55502"
ts=2026-05-05T11:27:18.857220558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:27:20.589234897Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:27:22.164829173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:27:22.520658663Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:27:25.934397884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:43796"
level=warn ts=2026-05-05T11:27:27.165091178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:27:32.164504529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:27:37.164882866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:27:42.164963953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:27:47.164025722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:27:47.164076162Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:27:47.164090656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:27:47.164102555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:27:48.859393865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:27:49.896193381Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:27:52.164679114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:27:53.539679392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T11:27:53.541447632Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:27:57.164077624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:27:58.729180432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:27:58.73621219Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:28:02.164287209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:28:07.165099027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:28:12.164602851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:28:17.144246348Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:28:17.144625466Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:28:17.164991855Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:28:17.165023991Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:28:17.165033014Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:28:17.165022489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:28:18.861624043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:28:19.2121885Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:28:22.165118289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:28:27.164954667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:28:28.887889247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52842"
level=warn ts=2026-05-05T11:28:32.164712928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:28:37.164822745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:28:38.61618305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-05T11:28:42.165101811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:28:46.134454488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:28:46.141011817Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T11:28:47.164777374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:28:47.164806163Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:28:47.16481433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:28:47.164841918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:28:47.861178393Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T11:28:47.994700035Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:43882"
ts=2026-05-05T11:28:48.864589948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=warn ts=2026-05-05T11:28:52.16476356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:28:57.164809433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:29:02.164595413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:29:07.165011724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:29:12.165115678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:29:15.101611599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:29:17.144181627Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:29:17.144635625Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=warn ts=2026-05-05T11:29:17.164531908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:29:17.164553985Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:29:17.164592311Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:29:17.164606483Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T11:29:18.866768773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=warn ts=2026-05-05T11:29:22.164695484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:29:27.164923414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:29:28.894794759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:59854"
level=info ts=2026-05-05T11:29:32.091007412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:29:32.097351238Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:29:32.16500696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:29:37.164740539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:29:42.164768073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:29:42.535660327Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:29:47.164468454Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:29:47.164499493Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:29:47.164518985Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:29:47.164521507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:29:48.342894264Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-05T11:29:48.868462287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=warn ts=2026-05-05T11:29:52.164427637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:29:57.164723216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:30:02.164830775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:30:07.164128584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:30:09.812474408Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:30:12.165146738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:30:16.148920331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:30:16.157550625Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T11:30:17.143889643Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:30:17.144188674Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=debug ts=2026-05-05T11:30:17.145440728Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-05T11:30:17.145476688Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975533 n=1
level=debug ts=2026-05-05T11:30:17.145517625Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975533 n=1
level=info ts=2026-05-05T11:30:17.164523204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:30:17.164551922Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:30:17.164562546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T11:30:17.164570096Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-05T11:30:17.164585755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:30:17.165028464Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-05T11:30:17.165045144Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T11:30:17.165098184Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=46.369µs success=true
level=debug ts=2026-05-05T11:30:17.165387786Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-05T11:30:18.002499961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:34558"
ts=2026-05-05T11:30:18.87064675Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:30:21.520701767Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-05T11:30:22.164600847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:30:25.876079913Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-05T11:30:27.164488423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:30:30.23079841Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-05T11:30:32.165034493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:30:34.585720134Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=warn ts=2026-05-05T11:30:37.164390263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:30:38.940396039Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=warn ts=2026-05-05T11:30:42.164453171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:30:43.294685944Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=debug ts=2026-05-05T11:30:43.695915443Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:30:45.278627739Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-05-05T11:30:47.164836036Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:30:47.164864388Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T11:30:47.164860753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:30:47.164885022Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T11:30:47.650378557Z caller=checkpoint.go:372 msg="writing series" size="408 kB"
ts=2026-05-05T11:30:48.873235249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:30:52.004704751Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-05T11:30:52.164645657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:30:56.359694634Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=warn ts=2026-05-05T11:30:57.164635582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:30:58.904696445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57582"
level=info ts=2026-05-05T11:30:59.120292843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:30:59.128418789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:31:00.714801972Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-05T11:31:00.714827507Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-05T11:31:02.164289723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:05.069137442Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-05T11:31:07.164826231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:08.415158485Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:31:09.424161888Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=warn ts=2026-05-05T11:31:12.165091151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:13.778298584Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-05T11:31:17.143896124Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:31:17.144265369Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:31:17.164580602Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:31:17.16461319Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:31:17.164623605Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:31:17.16463272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:31:18.005931024Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54830"
level=debug ts=2026-05-05T11:31:18.133881311Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-05T11:31:18.87539871Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=warn ts=2026-05-05T11:31:22.164762622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:22.488660701Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=debug ts=2026-05-05T11:31:26.843325312Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-05T11:31:27.164921812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:31.197924675Z caller=checkpoint.go:372 msg="writing series" size="344 kB"
level=warn ts=2026-05-05T11:31:32.164297576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:34.868841492Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:31:35.552854365Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-05T11:31:37.164966462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:39.907547974Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T11:31:42.164773114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:31:43.928498008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:31:43.935830931Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:31:44.262912762Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T11:31:47.164144681Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:31:47.164177427Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:31:47.164186701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:31:47.164195653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:48.617716433Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
ts=2026-05-05T11:31:48.877387224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=warn ts=2026-05-05T11:31:52.164568841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:52.972067597Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-05T11:31:57.165230177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:31:57.327561056Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=debug ts=2026-05-05T11:31:57.327586916Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=debug ts=2026-05-05T11:31:57.431425179Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-05T11:32:01.685155128Z caller=checkpoint.go:372 msg="writing series" size="5.2 MB"
level=debug ts=2026-05-05T11:32:01.68517836Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.2 MB" series=1
level=debug ts=2026-05-05T11:32:01.778297753Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:32:02.16480708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:32:06.036943577Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=warn ts=2026-05-05T11:32:07.16456101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:32:10.392207818Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=warn ts=2026-05-05T11:32:12.164293775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:32:14.746991677Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-05T11:32:17.143886664Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:32:17.144977639Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:32:17.164222979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:32:17.164247296Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:32:17.164255479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:32:17.164272396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:32:18.880189202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:32:19.101086514Z caller=checkpoint.go:372 msg="writing series" size="8.4 kB"
level=warn ts=2026-05-05T11:32:22.16486777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:32:23.456498426Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-05T11:32:26.476134914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:32:26.483951629Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:32:26.48397122Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:32:26.483995282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:32:26.490290484Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:32:27.164934145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:32:27.811357007Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T11:32:32.164190044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:32:32.166390333Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-05T11:32:35.47940856Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:32:36.521226228Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-05T11:32:37.16507182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:32:38.825473471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:32:38.834149497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:32:40.876176Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-05T11:32:42.165013236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:32:45.230279073Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=info ts=2026-05-05T11:32:47.164924789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:32:47.164954424Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:32:47.164962664Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:32:47.164993388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:32:48.012246689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36602"
ts=2026-05-05T11:32:48.882415234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:32:49.585178854Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-05T11:32:49.585201629Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-05T11:32:52.164301527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:32:53.940596146Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
ts=2026-05-05T11:32:55.961825041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:57818"
level=warn ts=2026-05-05T11:32:57.165012223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:32:58.295025283Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-05T11:33:02.164441481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:02.650340598Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
level=info ts=2026-05-05T11:33:05.684565605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:33:05.692590725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:33:07.004648027Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-05T11:33:07.164368557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:10.20871895Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:33:10.98634009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-05T11:33:10.987974817Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T11:33:11.359410169Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T11:33:12.164450435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:15.713804881Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-05T11:33:17.143672518Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:33:17.14472667Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T11:33:17.164108877Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:33:17.164137426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:33:17.16414915Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:33:17.164147866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:33:18.884649777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:33:20.069085811Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=warn ts=2026-05-05T11:33:22.165071894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:24.424237639Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
ts=2026-05-05T11:33:25.963880542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:39062"
level=warn ts=2026-05-05T11:33:27.164924989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:28.779115473Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-05T11:33:32.16471483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:33.134123069Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T11:33:37.164421703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:37.488549047Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-05T11:33:41.84332035Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-05T11:33:42.165161748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:44.398850389Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:33:46.19790329Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-05T11:33:47.164097549Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T11:33:47.164110141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:47.164133051Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:33:47.164143658Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T11:33:48.887311979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:33:50.553446377Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-05T11:33:52.165158165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:33:54.90816018Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
ts=2026-05-05T11:33:55.966810192Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:52358"
level=warn ts=2026-05-05T11:33:57.164349288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:33:58.922472594Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45028"
level=debug ts=2026-05-05T11:33:59.264339331Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-05T11:33:59.264362616Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=16
level=warn ts=2026-05-05T11:34:02.164442794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:34:03.617882337Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-05T11:34:05.573776537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:34:05.582239461Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:34:06.125420115Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T11:34:07.164723336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:34:07.972473193Z caller=checkpoint.go:372 msg="writing series" size="392 kB"
level=debug ts=2026-05-05T11:34:10.490904767Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:34:12.165123545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:34:12.327564726Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-05T11:34:16.681814337Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=debug ts=2026-05-05T11:34:16.681837529Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-05T11:34:17.144335542Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:34:17.145393824Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:34:17.164811088Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:34:17.164838307Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:34:17.164847087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:34:17.164873405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:34:18.890327146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:34:21.03659696Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-05T11:34:22.164970478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:34:25.392113132Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-05T11:34:27.164229343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:34:28.925365802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48140"
level=debug ts=2026-05-05T11:34:29.746658789Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-05T11:34:32.164482565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:34:34.101763966Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-05T11:34:36.943191804Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:34:37.164871947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:34:38.456744652Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=warn ts=2026-05-05T11:34:42.164229154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:34:42.810719752Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-05T11:34:47.164262551Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T11:34:47.164261172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:34:47.164311772Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:34:47.164320357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T11:34:47.165350165Z caller=checkpoint.go:384 msg="flushing series" totalSize="409 kB" series=6
level=info ts=2026-05-05T11:34:47.171230343Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-05T11:34:47.175395129Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010077967s
ts=2026-05-05T11:34:48.023289248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56214"
ts=2026-05-05T11:34:48.89263876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T11:34:49.908191173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:34:49.919662612Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:34:52.164782235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:34:57.164872928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:34:58.928585958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51280"
level=warn ts=2026-05-05T11:35:02.164374536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:35:07.164816471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:35:07.187746855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:35:10.84093148Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:35:12.16471746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:35:17.144015609Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:35:17.144047792Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:35:17.16452387Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T11:35:17.164531456Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:35:17.164570142Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:35:17.164581602Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:35:17.164573858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:35:17.164722711Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-05T11:35:17.164739766Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T11:35:17.164772386Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=25.903µs success=true
level=debug ts=2026-05-05T11:35:17.165085167Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-05T11:35:18.894380991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:35:21.319180896Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-05T11:35:22.164858166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:35:25.473067675Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=warn ts=2026-05-05T11:35:27.164331395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:35:29.627529237Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-05T11:35:32.165007248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:35:33.78053263Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-05T11:35:37.164621107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:35:37.706268549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:35:37.706307974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-05T11:35:37.720049323Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:35:37.934946953Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=debug ts=2026-05-05T11:35:42.088572696Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-05T11:35:42.165198449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:35:46.242710979Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=debug ts=2026-05-05T11:35:46.242747316Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-05T11:35:47.164658306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:35:47.164694491Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:35:47.16470256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:35:47.164751529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:35:48.028444748Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56888"
ts=2026-05-05T11:35:48.897352314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T11:35:48.937030544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:35:48.945519452Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:35:49.43894303Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T11:35:49.657525393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:35:49.665364785Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:35:50.396556249Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-05-05T11:35:52.164568963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:35:54.549750424Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=warn ts=2026-05-05T11:35:57.164526945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:35:58.704100853Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-05T11:36:02.164287348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:36:02.857707465Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=debug ts=2026-05-05T11:36:07.012449979Z caller=checkpoint.go:372 msg="writing series" size="366 kB"
level=warn ts=2026-05-05T11:36:07.16454462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:36:08.13830877Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:36:11.165670094Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-05T11:36:12.164662258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:36:15.319110838Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-05T11:36:17.144044695Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:36:17.146224863Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:36:17.164704404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:36:17.164730391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:36:17.164743039Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:36:17.164754133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:36:18.431472327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:36:18.443019298Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-05T11:36:18.900401003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:36:19.472809476Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=warn ts=2026-05-05T11:36:22.165496709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:36:23.627442864Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=debug ts=2026-05-05T11:36:23.627477836Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-05T11:36:27.164409158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:36:27.788137714Z caller=checkpoint.go:372 msg="writing series" size="9.2 MB"
level=debug ts=2026-05-05T11:36:27.788176958Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.2 MB" series=1
level=debug ts=2026-05-05T11:36:31.935248452Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-05T11:36:32.16501228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:36:36.088147032Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T11:36:37.164519925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:36:40.242508254Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-05T11:36:42.16492828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:36:42.256266921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:36:42.263747259Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:36:42.758554256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:36:43.275934337Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:36:44.395983783Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-05-05T11:36:47.164183922Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T11:36:47.164198058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:36:47.164217359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:36:47.164225994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T11:36:48.550238182Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-05-05T11:36:48.903323084Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=warn ts=2026-05-05T11:36:52.164660716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:36:52.703956278Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=debug ts=2026-05-05T11:36:56.858270535Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-05T11:36:57.164697361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:01.014035572Z caller=checkpoint.go:372 msg="writing series" size="5.8 MB"
level=debug ts=2026-05-05T11:37:01.014077944Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.6 MB" series=8
level=warn ts=2026-05-05T11:37:02.164329032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:05.165896256Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-05T11:37:07.165136692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:09.31920608Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=warn ts=2026-05-05T11:37:12.165121319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:13.473604988Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-05T11:37:13.475783229Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:37:14.050462709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:37:14.060450658Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T11:37:17.144357445Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:37:17.144761165Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:37:17.164243267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T11:37:17.164248669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:17.164266076Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:37:17.164275298Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T11:37:17.627579762Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
ts=2026-05-05T11:37:18.037332627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57998"
ts=2026-05-05T11:37:18.905952637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:37:21.781495166Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=warn ts=2026-05-05T11:37:22.165030624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:25.935277555Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=warn ts=2026-05-05T11:37:27.164972612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:30.088882626Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-05T11:37:31.993398523Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:37:31.99607266Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T11:37:32.164646773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:37:32.482832914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:37:32.489738563Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:37:34.242391744Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-05T11:37:37.164125382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:38.396720906Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-05-05T11:37:42.164262799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:42.550076484Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=debug ts=2026-05-05T11:37:42.889246437Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:37:46.704738797Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-05T11:37:47.164655127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:37:47.164712168Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:37:47.164726639Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:37:47.164737193Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T11:37:48.908811019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:37:50.858126728Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-05T11:37:52.164969412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:55.011572874Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-05T11:37:57.164551437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:37:59.165394197Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-05-05T11:37:59.16542382Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-05-05T11:38:02.164964014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:38:03.319682228Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-05T11:38:07.164346042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:38:07.473131465Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T11:38:08.016951385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:38:08.027358071Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:38:11.627460572Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-05-05T11:38:11.749882715Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:38:12.165135007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:38:15.781103399Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-05T11:38:17.143849457Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:38:17.143881261Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T11:38:17.164389644Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:38:17.164411227Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:38:17.164419958Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:38:17.164453423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:38:18.041437031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60286"
ts=2026-05-05T11:38:18.910975654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:38:19.935030301Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-05T11:38:22.164337192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:38:24.089039235Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-05T11:38:27.164806105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:38:27.709952733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:38:27.718873844Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:38:28.242225174Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-05T11:38:30.702674087Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T11:38:32.164121788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:38:32.397686515Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-05T11:38:36.550012786Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T11:38:37.164655217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:38:40.704619623Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-05T11:38:42.164434917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:38:44.858047789Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-05T11:38:45.865814608Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:38:47.164802785Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:38:47.16484126Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:38:47.164852824Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:38:47.164899668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:38:48.042561794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:43390"
ts=2026-05-05T11:38:48.913018337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:38:49.011284583Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-05T11:38:51.553248011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:38:51.562008441Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T11:38:52.164614585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:38:53.166136984Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=warn ts=2026-05-05T11:38:57.164929928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:38:57.32025309Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-05T11:38:57.320280981Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=14
level=debug ts=2026-05-05T11:39:01.473115662Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-05T11:39:02.16459113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:39:05.626979028Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-05T11:39:07.164755661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:39:09.781037496Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
level=info ts=2026-05-05T11:39:12.111270391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:39:12.121024461Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:39:12.121066556Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:39:12.121101139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:39:12.128771888Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:39:12.164994962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:39:13.935102055Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-05T11:39:14.813105056Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T11:39:17.143945292Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:39:17.144169259Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:39:17.164692091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:39:17.164729704Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:39:17.164744049Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:39:17.164746311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:39:18.088652428Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
ts=2026-05-05T11:39:18.914576107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:39:19.973402652Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:39:22.164649693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:39:22.242125309Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-05T11:39:26.397013856Z caller=checkpoint.go:372 msg="writing series" size="384 kB"
level=debug ts=2026-05-05T11:39:26.397045137Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=warn ts=2026-05-05T11:39:27.164658662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:39:28.365618141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:39:28.378223235Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T11:39:28.954307979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53124"
level=debug ts=2026-05-05T11:39:30.550388778Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-05T11:39:32.164481203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:39:32.568654367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="94 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:39:32.578082028Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:39:34.704181619Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-05T11:39:37.164591233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:39:38.857831953Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-05T11:39:42.164733472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:39:43.012327875Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-05-05T11:39:44.229701483Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:39:47.164025974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:39:47.164099973Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:39:47.164120623Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:39:47.164119767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:39:47.165111925Z caller=checkpoint.go:384 msg="flushing series" totalSize="481 kB" series=4
level=info ts=2026-05-05T11:39:47.273110224Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-05T11:39:47.27582625Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.110842348s
ts=2026-05-05T11:39:48.915694251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T11:39:52.066329531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:39:52.074695166Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:39:52.165165938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:39:57.164931882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:40:02.165249053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:40:07.164809224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:40:12.164725801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:40:13.450690263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:40:13.457219665Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:40:14.79797996Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T11:40:17.143898038Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:40:17.143927344Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:40:17.164438225Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:40:17.16445686Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:40:17.16445066Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T11:40:17.164473614Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:40:17.164450775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:40:17.164554736Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-05T11:40:17.164563574Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T11:40:17.164581446Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.97µs success=true
level=debug ts=2026-05-05T11:40:17.164764354Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-05T11:40:18.048866834Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60454"
level=debug ts=2026-05-05T11:40:18.665484175Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T11:40:18.917333941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:40:20.672140876Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-05T11:40:22.165088663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:40:24.178096451Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-05T11:40:27.164344287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:40:27.685032125Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
ts=2026-05-05T11:40:28.960164426Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47314"
level=debug ts=2026-05-05T11:40:31.191163587Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-05T11:40:32.164947549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:40:34.697972253Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-05-05T11:40:37.165168528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:40:38.204790644Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-05T11:40:41.710420113Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-05T11:40:42.16448444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:40:42.71309999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:40:42.722465653Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:40:45.217994312Z caller=checkpoint.go:372 msg="writing series" size="545 kB"
level=debug ts=2026-05-05T11:40:45.218017609Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-05-05T11:40:47.164089286Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T11:40:47.164102733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:40:47.164116601Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:40:47.164125792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T11:40:47.842762729Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T11:40:48.050370278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57004"
level=debug ts=2026-05-05T11:40:48.723768812Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
ts=2026-05-05T11:40:48.918553009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=warn ts=2026-05-05T11:40:52.164850553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:40:52.230121691Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=debug ts=2026-05-05T11:40:55.736408265Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-05T11:40:57.164118281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:40:59.243502685Z caller=checkpoint.go:372 msg="writing series" size="394 kB"
level=warn ts=2026-05-05T11:41:02.164344291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:02.749665572Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-05T11:41:06.256210065Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-05T11:41:07.054905752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:41:07.064721846Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T11:41:07.164687835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:09.762797966Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-05T11:41:12.165132454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:13.269145593Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-05T11:41:13.650103122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:41:14.749857453Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:41:16.775224541Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-05T11:41:17.143921526Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:41:17.143938067Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:41:17.16440139Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:41:17.164425816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:41:17.164433837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:41:17.164471768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:41:18.920838638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:41:20.282493358Z caller=checkpoint.go:372 msg="writing series" size="509 kB"
level=debug ts=2026-05-05T11:41:20.282517311Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=10
level=warn ts=2026-05-05T11:41:22.164767849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:23.805189624Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-05T11:41:23.805231595Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=1
level=warn ts=2026-05-05T11:41:27.164460174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:27.298307959Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
ts=2026-05-05T11:41:28.966556021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45038"
level=debug ts=2026-05-05T11:41:30.802512712Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T11:41:32.16513102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:34.307940608Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-05T11:41:37.164598327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:37.814175669Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-05T11:41:39.64304019Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:41:39.643131086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:41:39.647728735Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:41:41.32113462Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-05T11:41:41.887961365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:41:41.898391562Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:41:42.164611718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:44.827520865Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-05T11:41:47.165026929Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:41:47.165082003Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:41:47.165092557Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:41:47.165137144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:48.334508692Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-05T11:41:48.926198628Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T11:41:49.840441383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:41:49.852191255Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:41:49.852224629Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:41:49.852264416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:41:49.859765848Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:41:51.840638235Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-05-05T11:41:52.164863153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:55.346878316Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-05T11:41:56.570337085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:41:56.574358514Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T11:41:57.165028191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:41:58.853799747Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-05T11:42:02.164696257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:02.364630903Z caller=checkpoint.go:372 msg="writing series" size="6.6 MB"
level=debug ts=2026-05-05T11:42:02.364652715Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.6 MB" series=11
level=info ts=2026-05-05T11:42:03.780660125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:42:03.788374527Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:42:05.866264531Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=warn ts=2026-05-05T11:42:07.164792259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:08.72072345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:42:08.722860296Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T11:42:09.373132141Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=debug ts=2026-05-05T11:42:10.393184063Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:42:12.164749709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:12.880048091Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-05T11:42:16.386407742Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-05-05T11:42:17.143531603Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:42:17.144590528Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T11:42:17.164177577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:42:17.164200698Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:42:17.164209194Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:42:17.164210347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:42:18.058713992Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:38798"
ts=2026-05-05T11:42:18.928649844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:42:19.89236765Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=warn ts=2026-05-05T11:42:22.164713808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:23.399602649Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-05T11:42:26.018922356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:42:26.025864406Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T11:42:26.117112148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:42:26.124251996Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:42:26.905657533Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T11:42:27.164721756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:30.41168472Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-05T11:42:32.165138369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:33.918835357Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-05T11:42:36.305558415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:42:36.311132062Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T11:42:37.164954684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:37.425232422Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-05-05T11:42:40.932069357Z caller=checkpoint.go:372 msg="writing series" size="396 kB"
level=debug ts=2026-05-05T11:42:40.932099273Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=warn ts=2026-05-05T11:42:42.165110967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:42.227353094Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:42:44.438553697Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=info ts=2026-05-05T11:42:47.164903746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:42:47.164937266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:42:47.164945871Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:42:47.164969022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:47.944669585Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
ts=2026-05-05T11:42:48.931557013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:42:51.451289841Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-05T11:42:52.164743478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:54.141227173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:42:54.9579464Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
ts=2026-05-05T11:42:56.012769623Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:35712"
level=warn ts=2026-05-05T11:42:57.164917855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:42:58.46434186Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-05T11:43:01.971014281Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-05T11:43:02.164774124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:05.476683541Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-05T11:43:06.903360024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:43:06.912352649Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T11:43:07.164960387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:08.983816921Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-05T11:43:10.789129032Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:43:12.165034005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:12.490100682Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-05T11:43:15.996372406Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-05T11:43:17.144297956Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:43:17.14430049Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T11:43:17.164815211Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:43:17.164848615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:43:17.16485664Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:43:17.164888871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:43:18.274280329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:43:18.282206363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T11:43:18.933802989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:43:19.50249473Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-05T11:43:22.164733234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:23.009833171Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-05T11:43:26.516105796Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-05T11:43:27.165098478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:43:28.043689373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:43:28.048123071Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:43:30.022750516Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T11:43:32.165048307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:33.529398919Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=debug ts=2026-05-05T11:43:33.529422279Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=15
level=debug ts=2026-05-05T11:43:34.930490386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:43:37.035307815Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T11:43:37.164579834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:39.035328706Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:43:40.542482832Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-05T11:43:42.164307923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:44.048710367Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-05T11:43:47.164668256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:43:47.164705785Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:43:47.164716455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:43:47.164729841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:47.554744867Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
ts=2026-05-05T11:43:48.065494631Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56778"
ts=2026-05-05T11:43:48.936584697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:43:51.061963404Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-05T11:43:52.165023998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:54.567596766Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
ts=2026-05-05T11:43:56.016719795Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:53524"
level=warn ts=2026-05-05T11:43:57.164544221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:43:58.073930716Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-05T11:44:01.581203067Z caller=checkpoint.go:372 msg="writing series" size="418 kB"
level=warn ts=2026-05-05T11:44:02.164654539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:44:05.088655849Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-05T11:44:05.08868394Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=9
level=info ts=2026-05-05T11:44:05.218957244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="165 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:44:05.229991425Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:44:06.310267501Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:44:07.164681137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:44:08.593381516Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=debug ts=2026-05-05T11:44:12.100739624Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-05T11:44:12.164118504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:44:14.650980522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:44:14.657628613Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:44:15.607210026Z caller=checkpoint.go:372 msg="writing series" size="724 kB"
level=info ts=2026-05-05T11:44:17.143974445Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:44:17.144087661Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:44:17.164728408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:44:17.16475263Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:44:17.164759975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:44:17.164770421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:44:18.938550823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:44:19.113917463Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=warn ts=2026-05-05T11:44:22.164815083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:44:22.620293948Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=debug ts=2026-05-05T11:44:22.620316311Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-05T11:44:23.467221004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:44:23.471154713Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:44:26.126572257Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-05T11:44:27.165585887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:44:29.632843957Z caller=checkpoint.go:372 msg="writing series" size="583 kB"
level=warn ts=2026-05-05T11:44:32.164810645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:44:33.139004322Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-05T11:44:35.015747076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:44:35.906287509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:44:36.646095191Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=warn ts=2026-05-05T11:44:37.165022232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:44:40.152263364Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-05T11:44:40.152293206Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-05-05T11:44:42.164884968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:44:43.658992405Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-05-05T11:44:47.164031736Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:44:47.164084776Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:44:47.164093417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:44:47.164104029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:44:47.165094071Z caller=checkpoint.go:384 msg="flushing series" totalSize="209 kB" series=1
level=info ts=2026-05-05T11:44:47.200870515Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-05T11:44:47.203658447Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.038952532s
ts=2026-05-05T11:44:48.939855607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=warn ts=2026-05-05T11:44:52.164521671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:44:56.022375235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:49784"
level=info ts=2026-05-05T11:44:56.317728664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="189 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:44:56.326372402Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T11:44:57.164448891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:45:01.446119104Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:45:02.16510807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:45:07.164450629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:45:12.164639063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:45:12.740839463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:45:12.749022283Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:45:17.143442525Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:45:17.144490043Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:45:17.144740168Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-05T11:45:17.144803473Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975534 n=1
level=debug ts=2026-05-05T11:45:17.144851462Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975534 n=1
level=info ts=2026-05-05T11:45:17.164487967Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:45:17.164524403Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:45:17.164537116Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T11:45:17.164568183Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-05T11:45:17.164564174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:45:17.164718935Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-05T11:45:17.164726247Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T11:45:17.164745758Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.687µs success=true
level=debug ts=2026-05-05T11:45:17.164967048Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-05T11:45:18.757397529Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:45:18.94248534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:45:20.671821494Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=info ts=2026-05-05T11:45:21.805169023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:45:21.810349525Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T11:45:22.164752914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:45:24.178633434Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-05T11:45:26.02481059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:55388"
level=debug ts=2026-05-05T11:45:27.07110171Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:45:27.165106133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:45:27.685221629Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=debug ts=2026-05-05T11:45:31.19156094Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-05T11:45:32.16453606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:45:34.698388138Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-05T11:45:37.165025903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:45:37.652733136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="189 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:45:37.663508048Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:45:38.204213843Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-05T11:45:41.711201655Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-05T11:45:42.164449472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:45:45.218454854Z caller=checkpoint.go:372 msg="writing series" size="485 kB"
level=debug ts=2026-05-05T11:45:45.218485189Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-05T11:45:47.164429585Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:45:47.16447158Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T11:45:47.164456994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:45:47.164480978Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T11:45:48.724359238Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
ts=2026-05-05T11:45:48.945565274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=warn ts=2026-05-05T11:45:52.164727394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:45:52.230910198Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=debug ts=2026-05-05T11:45:53.963718759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:45:53.965454018Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T11:45:55.053750777Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:45:55.736476573Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-05T11:45:57.164544174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:45:58.993712395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54266"
level=debug ts=2026-05-05T11:45:59.244430427Z caller=checkpoint.go:372 msg="writing series" size="952 kB"
level=debug ts=2026-05-05T11:45:59.244455297Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-05T11:46:01.669967055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:46:01.678401569Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:46:01.678432398Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:46:01.678460826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:46:01.685847212Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:46:02.164843567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:46:02.750566201Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=info ts=2026-05-05T11:46:03.62445916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:46:03.63127699Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:46:06.255979174Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-05T11:46:07.165223072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:46:09.763260188Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-05T11:46:12.164801538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:46:13.26997833Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-05T11:46:16.775979348Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-05T11:46:17.144226646Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:46:17.145552981Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:46:17.164940511Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:46:17.164964998Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:46:17.164973804Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:46:17.164983873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:46:18.948088567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T11:46:19.312854512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:46:19.324676603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:46:20.28328321Z caller=checkpoint.go:372 msg="writing series" size="815 kB"
level=debug ts=2026-05-05T11:46:20.283315432Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=debug ts=2026-05-05T11:46:20.787513126Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:46:22.164569278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:46:22.23072531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="213 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:46:22.238827837Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:46:23.820892209Z caller=checkpoint.go:372 msg="writing series" size="28 MB"
level=debug ts=2026-05-05T11:46:23.820919732Z caller=checkpoint.go:384 msg="flushing series" totalSize="28 MB" series=1
ts=2026-05-05T11:46:26.029131504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:41280"
level=warn ts=2026-05-05T11:46:27.164692217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:46:27.297361185Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=debug ts=2026-05-05T11:46:30.803123429Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T11:46:32.165014401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:46:34.308228303Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-05T11:46:37.164635044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:46:37.814334407Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-05T11:46:41.321593867Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=warn ts=2026-05-05T11:46:42.165083597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:46:44.828424214Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-05T11:46:47.164659293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:46:47.164650391Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:46:47.164685552Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:46:47.164694406Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T11:46:48.334332155Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
ts=2026-05-05T11:46:48.950426559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:46:50.510275256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:46:51.840760211Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=warn ts=2026-05-05T11:46:52.165008823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:46:52.839952897Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T11:46:53.313342953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:46:53.320919583Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:46:55.347375645Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-05T11:46:57.165014363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:46:58.854148875Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=debug ts=2026-05-05T11:46:58.854173629Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
ts=2026-05-05T11:46:59.00052181Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:47976"
level=warn ts=2026-05-05T11:47:02.16414748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:02.369405927Z caller=checkpoint.go:372 msg="writing series" size="7.3 MB"
level=debug ts=2026-05-05T11:47:02.369437422Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.3 MB" series=1
level=info ts=2026-05-05T11:47:02.69688464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:47:02.703428184Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:47:05.866561454Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-05T11:47:07.164526747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:09.37346035Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=warn ts=2026-05-05T11:47:12.165125108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:12.879923037Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-05T11:47:14.933514581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="237 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:47:14.945304112Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:47:16.386255366Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-05-05T11:47:17.143895259Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:47:17.14454392Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:47:17.164127511Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:47:17.164168296Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:47:17.16418065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:47:17.164209938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:18.305180654Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T11:47:18.953654056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:47:19.893106874Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=warn ts=2026-05-05T11:47:22.165900897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:23.39921473Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-05T11:47:26.905789524Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T11:47:27.164597795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:47:29.003758889Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57350"
level=debug ts=2026-05-05T11:47:30.412175168Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=warn ts=2026-05-05T11:47:32.165011659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:33.919178398Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-05T11:47:37.164394479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:37.337269664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:47:37.425172189Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=debug ts=2026-05-05T11:47:37.425217185Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=debug ts=2026-05-05T11:47:40.931880218Z caller=checkpoint.go:372 msg="writing series" size="381 kB"
level=warn ts=2026-05-05T11:47:42.16480701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:47:42.584001652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:47:42.594409205Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:47:44.43882348Z caller=checkpoint.go:372 msg="writing series" size="365 kB"
level=info ts=2026-05-05T11:47:47.164406017Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:47:47.164447603Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:47:47.164457914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:47:47.164483699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:47.944920101Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
ts=2026-05-05T11:47:48.955911315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T11:47:50.775137808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:47:50.779777199Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:47:51.451560544Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-05T11:47:52.164681802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:54.140147773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:47:54.957759793Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-05T11:47:57.164637822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:47:58.464596188Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-05T11:48:01.970789341Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-05-05T11:48:01.970812115Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-05T11:48:02.164464429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:05.476741162Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T11:48:07.164446316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:08.984233385Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T11:48:10.593959603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:48:10.609463538Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:48:10.609524626Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:48:10.609591857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:48:10.618637123Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T11:48:12.164351553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:12.490317993Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-05T11:48:15.996534Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-05T11:48:17.143767429Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:48:17.144167948Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T11:48:17.164367473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:48:17.164394062Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:48:17.164403771Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:48:17.164412827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:17.962558335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:48:18.957280643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:48:19.506215414Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-05T11:48:20.876963057Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:48:22.164402259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:23.009479235Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-05T11:48:26.516210391Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-05T11:48:26.839368467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:48:26.846254405Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T11:48:27.164549489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:30.02302315Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T11:48:32.164998933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:33.529306436Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-05T11:48:34.094264182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:48:34.107492752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:48:37.036177813Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T11:48:37.165146006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:48:39.20673459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:48:39.211490328Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:48:39.211521416Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:48:39.211554444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:48:39.21554269Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:48:40.541962833Z caller=checkpoint.go:372 msg="writing series" size="440 kB"
level=warn ts=2026-05-05T11:48:42.164948046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:44.048331291Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-05-05T11:48:44.048362887Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-05-05T11:48:47.164022292Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:48:47.164096776Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:48:47.16411635Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:48:47.164123283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:47.554809377Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
ts=2026-05-05T11:48:48.960113001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:48:51.06115392Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-05T11:48:51.298371132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:48:51.298445112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:48:51.31029595Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T11:48:51.649831948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:48:51.65390711Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T11:48:52.165165429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:48:52.926498594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:48:52.933254524Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:48:54.574347221Z caller=checkpoint.go:372 msg="writing series" size="4.3 MB"
level=debug ts=2026-05-05T11:48:54.574370303Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=3
level=warn ts=2026-05-05T11:48:57.164669814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:48:58.074976704Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=debug ts=2026-05-05T11:49:01.581067182Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=warn ts=2026-05-05T11:49:02.164333379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:49:05.089159104Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-05T11:49:05.089195327Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=info ts=2026-05-05T11:49:06.033290062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:49:06.042803746Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T11:49:07.16469485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:49:08.595095222Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=debug ts=2026-05-05T11:49:12.101137036Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-05T11:49:12.164904955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:49:13.531688958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:49:13.536905168Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:49:15.607891241Z caller=checkpoint.go:372 msg="writing series" size="870 kB"
level=info ts=2026-05-05T11:49:17.143994758Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:49:17.144036683Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:49:17.164403864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:49:17.16442585Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:49:17.164434134Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:49:17.164423995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:49:18.094440325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54650"
level=debug ts=2026-05-05T11:49:18.294770676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:49:18.963258842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:49:19.113791329Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-05-05T11:49:19.113832106Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-05T11:49:22.164623724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:49:22.19987928Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:49:22.62026081Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-05T11:49:25.059106559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:49:25.067108173Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T11:49:26.046408647Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:57450"
level=debug ts=2026-05-05T11:49:26.126829044Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-05T11:49:27.164707661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:49:29.632988819Z caller=checkpoint.go:372 msg="writing series" size="819 kB"
level=debug ts=2026-05-05T11:49:29.633011534Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-05-05T11:49:32.164714324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:49:33.13932743Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-05T11:49:36.645942076Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=warn ts=2026-05-05T11:49:37.164981303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:49:39.497320277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:49:39.512264535Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T11:49:40.152223125Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=info ts=2026-05-05T11:49:41.717691845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:49:41.727558345Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T11:49:42.16429059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:49:43.65914892Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-05T11:49:47.164725291Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:49:47.164761348Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:49:47.164772806Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:49:47.164784201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:49:47.165860248Z caller=checkpoint.go:384 msg="flushing series" totalSize="466 kB" series=4
level=info ts=2026-05-05T11:49:47.169019011Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-05T11:49:47.171422323Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006522736s
ts=2026-05-05T11:49:48.096195742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59522"
ts=2026-05-05T11:49:48.966321215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:49:50.390029837Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:49:52.164340325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:49:57.164961093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:50:02.164567343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:50:06.483149577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:50:06.489625401Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T11:50:07.164636178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:50:08.605901684Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:50:08.608407863Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T11:50:10.33926972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:50:10.346571188Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:50:12.165117776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:50:16.943719512Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:50:17.144004234Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:50:17.14402897Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:50:17.164483271Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-05T11:50:17.164495226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:50:17.164503151Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:50:17.164527335Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:50:17.164540035Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T11:50:17.164671441Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-05T11:50:17.16468195Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T11:50:17.164704351Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.504µs success=true
level=debug ts=2026-05-05T11:50:17.164959711Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-05T11:50:18.098461091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52012"
ts=2026-05-05T11:50:18.969564064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:50:20.627544067Z caller=checkpoint.go:372 msg="writing series" size="949 kB"
level=warn ts=2026-05-05T11:50:22.164757927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:50:24.088483809Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-05-05T11:50:26.050576262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:46776"
level=warn ts=2026-05-05T11:50:27.16486695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:50:27.549974286Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-05T11:50:27.549996855Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-05T11:50:28.380271803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:50:28.38890355Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T11:50:31.011536068Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-05T11:50:32.165152188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:50:33.518266531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="284 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:50:33.525020629Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:50:34.473039376Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-05T11:50:37.164617261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:50:37.93470105Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=debug ts=2026-05-05T11:50:41.395833593Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-05T11:50:41.603446987Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:50:42.164844122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:50:44.858326035Z caller=checkpoint.go:372 msg="writing series" size="620 kB"
level=info ts=2026-05-05T11:50:47.164190326Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:50:47.164240182Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:50:47.164252909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:50:47.16427139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:50:48.099482547Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52282"
level=debug ts=2026-05-05T11:50:48.318962447Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=debug ts=2026-05-05T11:50:48.318992023Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-05T11:50:48.971607497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:50:51.781013024Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=warn ts=2026-05-05T11:50:52.164445884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:50:53.268431934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:50:53.278691427Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:50:55.242136629Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-05T11:50:57.164453252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:50:58.705634004Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-05T11:50:58.705658211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-05T11:51:01.759558578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:51:01.76657656Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T11:51:02.164693986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:02.166157693Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=debug ts=2026-05-05T11:51:05.626941533Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=warn ts=2026-05-05T11:51:07.164641233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:09.088516075Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-05T11:51:10.161383133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T11:51:10.729677049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:51:10.736380531Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T11:51:12.164670621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:12.550248777Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-05T11:51:13.608347759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:51:16.011183069Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-05T11:51:17.14387678Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:51:17.143908072Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:51:17.164330108Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:51:17.164354087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:51:17.164364143Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:51:17.164402109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:51:18.974341858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:51:19.474020301Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-05T11:51:19.474043311Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=6
level=warn ts=2026-05-05T11:51:22.164990222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:51:22.602738979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:51:22.610410703Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:51:22.934401321Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-05T11:51:26.47740659Z caller=checkpoint.go:372 msg="writing series" size="38 MB"
level=debug ts=2026-05-05T11:51:26.477449836Z caller=checkpoint.go:384 msg="flushing series" totalSize="38 MB" series=2
level=warn ts=2026-05-05T11:51:27.164544591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:29.857579932Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-05T11:51:32.164447048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:33.319187598Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-05T11:51:36.780741344Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-05T11:51:37.165000614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:40.242957907Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T11:51:41.22212745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:51:41.231932708Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:51:42.164517652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:43.703743961Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-05T11:51:47.164805343Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T11:51:47.164806343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:47.164842847Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:51:47.164867947Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T11:51:47.166204976Z caller=checkpoint.go:372 msg="writing series" size="404 kB"
ts=2026-05-05T11:51:48.103307604Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54430"
level=debug ts=2026-05-05T11:51:48.376702883Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T11:51:48.976999719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:51:50.627589007Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-05T11:51:52.165130493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:54.088633031Z caller=checkpoint.go:372 msg="writing series" size="377 kB"
level=debug ts=2026-05-05T11:51:54.088659211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-05T11:51:54.269106838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:51:54.274366387Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T11:51:54.325469243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:51:54.33275523Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
ts=2026-05-05T11:51:56.058015524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34688"
level=warn ts=2026-05-05T11:51:57.164856182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:51:57.550008988Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-05T11:51:59.964948327Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:52:01.011603368Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=warn ts=2026-05-05T11:52:02.164848787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:52:04.070690525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:52:04.078782644Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:52:04.480557075Z caller=checkpoint.go:372 msg="writing series" size="7.7 MB"
level=debug ts=2026-05-05T11:52:04.480584915Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.3 MB" series=3
level=warn ts=2026-05-05T11:52:07.164735533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:52:07.934878764Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=debug ts=2026-05-05T11:52:11.396934963Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=warn ts=2026-05-05T11:52:12.164980599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:52:14.099301428Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:52:14.858117557Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-05T11:52:17.144087179Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:52:17.145265147Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:52:17.164669046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:52:17.164690975Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:52:17.164698835Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:52:17.16468546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:52:18.319789988Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
ts=2026-05-05T11:52:18.979509643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:52:21.780905046Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-05T11:52:22.164762958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:52:25.242498674Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-05-05T11:52:27.164703014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:52:28.704475183Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-05T11:52:29.970484553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:52:29.980438949Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:52:29.980465196Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:52:29.9804957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:52:29.986669535Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:52:32.164307613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:52:32.165381328Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-05T11:52:35.626820824Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-05T11:52:36.404713029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:52:36.411812795Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T11:52:36.529090149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:52:36.533377862Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T11:52:37.164121782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:52:39.089136985Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T11:52:39.089161916Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=debug ts=2026-05-05T11:52:41.193296429Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:52:41.291522445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:52:41.29974649Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:52:42.165111575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:52:42.549968051Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
level=debug ts=2026-05-05T11:52:46.012175266Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=warn ts=2026-05-05T11:52:47.164695846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:52:47.164694433Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:52:47.16473504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:52:47.164744486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T11:52:48.10779476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48306"
level=debug ts=2026-05-05T11:52:48.67953706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:52:48.981730909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:52:49.473729313Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-05T11:52:52.164658568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:52:52.934784091Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-05T11:52:52.934805954Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-05T11:52:56.063473434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:55982"
level=debug ts=2026-05-05T11:52:56.39654622Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-05T11:52:57.165182295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:52:58.000556446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="307 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:52:58.007933673Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:52:59.857698865Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-05T11:53:02.16482729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:03.319231705Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=debug ts=2026-05-05T11:53:06.780673216Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-05T11:53:07.164394416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:10.242783479Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T11:53:12.164123209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:13.019844975Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:53:13.704209541Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-05-05T11:53:16.946477245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:53:16.955622281Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T11:53:17.144322807Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:53:17.144704452Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:53:17.164149167Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:53:17.164174969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:53:17.164184311Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:53:17.164189747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:17.165958656Z caller=checkpoint.go:372 msg="writing series" size="358 kB"
ts=2026-05-05T11:53:18.984042968Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:53:20.627022863Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-05T11:53:22.164851124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:24.088266528Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-05-05T11:53:24.288032383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:53:24.31469928Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
ts=2026-05-05T11:53:26.066183416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:52722"
level=warn ts=2026-05-05T11:53:27.164594956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:27.549839858Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-05T11:53:27.703215917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:53:27.706861566Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T11:53:29.042870071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56156"
level=debug ts=2026-05-05T11:53:29.209418436Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:53:31.011569464Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T11:53:32.164366546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:34.473225724Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-05T11:53:37.164491502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:37.934840737Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-05T11:53:38.917752061Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:53:41.396407142Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=debug ts=2026-05-05T11:53:41.39643705Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=14
level=warn ts=2026-05-05T11:53:42.164914337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:44.857854757Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-05T11:53:47.164346612Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:53:47.164377089Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:53:47.164385178Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:53:47.164407723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:48.319698469Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
ts=2026-05-05T11:53:48.98624794Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:53:51.781035138Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-05T11:53:52.165091885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:53:55.247076646Z caller=checkpoint.go:372 msg="writing series" size="4.7 MB"
level=debug ts=2026-05-05T11:53:55.247104099Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.9 MB" series=4
level=warn ts=2026-05-05T11:53:57.164515425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:53:57.468563983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:53:57.478041066Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:53:58.703969028Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=info ts=2026-05-05T11:54:01.020749022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:54:01.028150627Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:54:02.164715855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:54:02.166221956Z caller=checkpoint.go:372 msg="writing series" size="748 kB"
level=debug ts=2026-05-05T11:54:05.630482565Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-05T11:54:05.630508477Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=3
level=warn ts=2026-05-05T11:54:07.164506724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:54:07.636737891Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:54:07.878489759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:54:07.885788471Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T11:54:09.089032654Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-05T11:54:12.16508681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:54:12.549815943Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-05-05T11:54:16.012002191Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T11:54:16.01202804Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-05T11:54:17.144712009Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:54:17.14535069Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:54:17.164788797Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:54:17.164817111Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:54:17.164828088Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:54:17.164836751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:54:17.93164392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:54:17.936024894Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T11:54:18.114325164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49080"
ts=2026-05-05T11:54:18.988575569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:54:19.473005018Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-05T11:54:22.1642632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:54:22.935283234Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
level=debug ts=2026-05-05T11:54:26.396701448Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-05T11:54:27.164785899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:54:27.576201744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:54:27.577786509Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T11:54:29.858577214Z caller=checkpoint.go:372 msg="writing series" size="990 kB"
level=debug ts=2026-05-05T11:54:29.85859961Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=warn ts=2026-05-05T11:54:32.164289803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:54:33.318871458Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-05T11:54:36.78112624Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=warn ts=2026-05-05T11:54:37.164635391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:54:39.779405213Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:54:40.242002473Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-05T11:54:42.164821103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:54:43.704315841Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=info ts=2026-05-05T11:54:44.634198999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:54:44.646546251Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:54:44.646578017Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:54:44.646631927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:54:44.654837688Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T11:54:47.164605368Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:54:47.164635306Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:54:47.164643245Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:54:47.164665856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:54:47.165680309Z caller=checkpoint.go:384 msg="flushing series" totalSize="739 kB" series=4
level=info ts=2026-05-05T11:54:47.172984081Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-05T11:54:47.175981167Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011092982s
ts=2026-05-05T11:54:48.991076175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=warn ts=2026-05-05T11:54:52.164974278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T11:54:57.164369053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:54:57.782498476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:54:57.790625808Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T11:55:02.164877106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:55:03.685339279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:55:03.691669718Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:55:04.099940151Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:55:04.684399791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T11:55:07.164985951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:55:07.803379867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:55:07.810225437Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:55:07.810251384Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:55:07.810281016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:55:07.817159054Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T11:55:11.054836976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:55:11.058346669Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:55:11.05837394Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:55:11.058404649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:55:11.061175589Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T11:55:12.164358549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:55:17.143612288Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:55:17.14468083Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T11:55:17.164173241Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-05T11:55:17.16418785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:55:17.164217082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:55:17.164248033Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:55:17.164259554Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T11:55:17.164303121Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-05T11:55:17.164311201Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T11:55:17.164375862Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=40.366µs success=true
level=debug ts=2026-05-05T11:55:17.164669117Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-05T11:55:18.117903047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:47900"
ts=2026-05-05T11:55:18.994028357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:55:20.583164733Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=warn ts=2026-05-05T11:55:22.164539713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:55:24.000788016Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-05T11:55:26.674756631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:55:26.684462423Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T11:55:27.164359878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:55:27.239760489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:55:27.249017811Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T11:55:27.387464875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:55:27.391115961Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:55:27.418269494Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-05T11:55:27.418293049Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-05T11:55:30.836401377Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-05T11:55:32.164321688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:55:34.25389816Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-05T11:55:37.165217338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:55:37.67192778Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-05T11:55:38.057182925Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T11:55:38.057242608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T11:55:38.065309009Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T11:55:41.089778441Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-05T11:55:42.164134001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:55:44.507046329Z caller=checkpoint.go:372 msg="writing series" size="639 kB"
level=debug ts=2026-05-05T11:55:44.507100258Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T11:55:47.164880158Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:55:47.164909703Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:55:47.164918152Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:55:47.164906465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:55:47.530111291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:55:47.537360226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:55:47.925294912Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
ts=2026-05-05T11:55:48.119816789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49490"
ts=2026-05-05T11:55:48.996180632Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:55:51.342792815Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=warn ts=2026-05-05T11:55:52.164576229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:55:53.956530044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T11:55:54.760046964Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
ts=2026-05-05T11:55:56.076439228Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:58862"
level=info ts=2026-05-05T11:55:56.699816482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T11:55:56.707317341Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T11:55:57.164863135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:55:57.767204529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:55:57.774248536Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T11:55:58.178937871Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-05T11:55:58.178960684Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=info ts=2026-05-05T11:55:58.439339818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:55:58.442604344Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:56:01.596220905Z caller=checkpoint.go:372 msg="writing series" size="385 kB"
level=warn ts=2026-05-05T11:56:02.164327474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:56:02.595911053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:56:05.013789344Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-05T11:56:07.165075131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:56:08.431444916Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-05T11:56:11.84894642Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=warn ts=2026-05-05T11:56:12.165044689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:56:15.26594168Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-05T11:56:17.144076789Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:56:17.14503818Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:56:17.164472083Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T11:56:17.164485756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:56:17.16449784Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:56:17.164507363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T11:56:18.120753133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49594"
level=debug ts=2026-05-05T11:56:18.685539781Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-05T11:56:18.685562061Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=6
ts=2026-05-05T11:56:18.99834985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:56:22.101750709Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-05T11:56:22.164697953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:56:23.579423652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:56:23.587803194Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:56:25.566918366Z caller=checkpoint.go:372 msg="writing series" size="44 MB"
level=debug ts=2026-05-05T11:56:25.566958508Z caller=checkpoint.go:384 msg="flushing series" totalSize="44 MB" series=2
level=warn ts=2026-05-05T11:56:27.164484615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:56:28.93709011Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=info ts=2026-05-05T11:56:31.237571787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:56:31.246113484Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:56:31.579757333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:56:32.164712459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:56:32.355524117Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-05T11:56:34.199205786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T11:56:34.208972189Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T11:56:35.772956895Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-05T11:56:37.164406455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:56:39.190467766Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-05T11:56:42.164181779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:56:42.608588434Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=debug ts=2026-05-05T11:56:46.026486542Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T11:56:46.026511409Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-05T11:56:46.134793453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:56:46.145238558Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T11:56:47.164595931Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:56:47.164643246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:56:47.164654157Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:56:47.164679116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:56:48.121976434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56230"
level=info ts=2026-05-05T11:56:48.53737381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:56:48.541028582Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T11:56:49.003030865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:56:49.444325616Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-05T11:56:52.164507179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:56:52.861981139Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=debug ts=2026-05-05T11:56:56.040544277Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T11:56:56.081211967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:45642"
level=debug ts=2026-05-05T11:56:56.279604157Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-05T11:56:57.085779027Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:56:57.164082387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:56:59.06468959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:52772"
level=debug ts=2026-05-05T11:56:59.696990422Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-05T11:57:02.164418368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:57:03.115138867Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T11:57:03.115173149Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-05-05T11:57:06.53736397Z caller=checkpoint.go:372 msg="writing series" size="8.7 MB"
level=debug ts=2026-05-05T11:57:06.537388965Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.7 MB" series=1
level=warn ts=2026-05-05T11:57:07.164783376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:57:09.950208063Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-05T11:57:11.934272243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:57:11.943378437Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T11:57:12.164520925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:57:13.368525342Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=debug ts=2026-05-05T11:57:16.785479155Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-05T11:57:17.144015524Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:57:17.144430541Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:57:17.164934098Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T11:57:17.164937549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:57:17.164956089Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:57:17.164963787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T11:57:19.00569873Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:57:20.202899937Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T11:57:22.164373732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:57:22.269735618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:57:22.277845548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T11:57:22.952062548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T11:57:22.960107749Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T11:57:23.150846408Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:57:23.621155102Z caller=checkpoint.go:372 msg="writing series" size="8.6 kB"
level=debug ts=2026-05-05T11:57:27.039081626Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=warn ts=2026-05-05T11:57:27.165038687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:57:30.456559351Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T11:57:32.164632562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:57:33.8745298Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-05T11:57:36.85385106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:57:36.862927973Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T11:57:37.164749064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:57:37.292225496Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-05T11:57:40.423822679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:57:40.432696363Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T11:57:40.709605442Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T11:57:40.709629081Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=debug ts=2026-05-05T11:57:41.062177842Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T11:57:42.165139748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:57:44.12766478Z caller=checkpoint.go:372 msg="writing series" size="469 kB"
level=warn ts=2026-05-05T11:57:47.164183995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:57:47.164191045Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:57:47.164233352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:57:47.164241971Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T11:57:47.545718645Z caller=checkpoint.go:372 msg="writing series" size="716 kB"
level=debug ts=2026-05-05T11:57:47.54574393Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
ts=2026-05-05T11:57:48.127006942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:46058"
ts=2026-05-05T11:57:49.009103136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T11:57:50.962210537Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-05T11:57:51.866932707Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:57:52.165072236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:57:52.93210418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:57:52.941124289Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:57:54.380408735Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-05T11:57:57.164101329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:57:57.797968897Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
ts=2026-05-05T11:57:59.070444061Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53408"
level=debug ts=2026-05-05T11:58:01.21569947Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-05T11:58:02.16464659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:04.634115776Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-05T11:58:07.164109968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:08.051532301Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-05T11:58:11.469553269Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T11:58:12.165003504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:14.886704535Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-05T11:58:17.144555917Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:58:17.146115023Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:58:17.164517233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:58:17.164543448Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T11:58:17.164531999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:58:17.164560795Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T11:58:17.948712021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:58:17.956417402Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T11:58:18.128131542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51194"
level=debug ts=2026-05-05T11:58:18.304381515Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=info ts=2026-05-05T11:58:18.515833536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T11:58:18.522976153Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-05T11:58:19.011774656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T11:58:19.398528513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:58:19.402987473Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:58:21.155661185Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:58:21.722577681Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-05T11:58:22.165189707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:25.140269169Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T11:58:25.14029728Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=11
level=warn ts=2026-05-05T11:58:27.165147823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:28.557343124Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=debug ts=2026-05-05T11:58:31.975619826Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-05T11:58:32.120954108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:58:32.131434382Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T11:58:32.165182449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:35.39291145Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-05T11:58:37.164719788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:38.81027966Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-05T11:58:38.915414902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T11:58:38.917291617Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T11:58:39.944026736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:58:39.950891828Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T11:58:42.164961686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:42.228752289Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=debug ts=2026-05-05T11:58:45.646796654Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-05T11:58:47.164196183Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:58:47.164229194Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:58:47.164242063Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:58:47.164277858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:58:49.012996316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T11:58:49.06414138Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-05T11:58:52.168326368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:52.481565297Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-05T11:58:52.481594905Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-05T11:58:53.228868029Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T11:58:55.904978605Z caller=checkpoint.go:372 msg="writing series" size="6.3 MB"
level=debug ts=2026-05-05T11:58:55.90500541Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.3 MB" series=1
level=warn ts=2026-05-05T11:58:57.164245411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:58:59.316858283Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-05T11:59:02.164772245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:59:02.735115001Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=debug ts=2026-05-05T11:59:06.154996339Z caller=checkpoint.go:372 msg="writing series" size="3.3 MB"
level=debug ts=2026-05-05T11:59:06.155023652Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=3
level=warn ts=2026-05-05T11:59:07.164353801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:59:09.570464933Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-05T11:59:12.16427768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:59:12.9878119Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-05T11:59:13.871208885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T11:59:13.878748368Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T11:59:14.042531694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T11:59:14.610541579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:59:14.618225577Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:59:14.618259196Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T11:59:14.618298536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:59:14.626617991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T11:59:15.670347852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T11:59:15.673511642Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T11:59:16.406872859Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-05T11:59:16.406897745Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-05T11:59:17.144031023Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T11:59:17.144109553Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T11:59:17.164577388Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:59:17.164604904Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:59:17.16461518Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T11:59:17.164619551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T11:59:18.132591318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48210"
ts=2026-05-05T11:59:19.015270771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T11:59:19.823609482Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-05T11:59:20.595322611Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T11:59:22.164991189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:59:23.241493946Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=info ts=2026-05-05T11:59:24.34520575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T11:59:24.355688791Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dlty\"}"
ts=2026-05-05T11:59:26.093530811Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:36982"
level=debug ts=2026-05-05T11:59:26.659334991Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-05T11:59:27.164200861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:59:30.077003739Z caller=checkpoint.go:372 msg="writing series" size="858 kB"
level=debug ts=2026-05-05T11:59:30.077027186Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-05T11:59:32.164753918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:59:33.494804866Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=info ts=2026-05-05T11:59:34.623450711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T11:59:34.633395428Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T11:59:36.912504512Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=warn ts=2026-05-05T11:59:37.164781729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:59:37.530762269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T11:59:37.538983037Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T11:59:40.330314342Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=warn ts=2026-05-05T11:59:42.164828573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:59:43.747279004Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-05T11:59:47.164999198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T11:59:47.164995766Z caller=checkpoint.go:384 msg="flushing series" totalSize="424 kB" series=4
level=warn ts=2026-05-05T11:59:47.165009714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:59:47.165028778Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T11:59:47.165038897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T11:59:47.168446137Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-05T11:59:47.172001858Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007391962s
level=debug ts=2026-05-05T11:59:47.812272804Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T11:59:48.134638806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:44984"
ts=2026-05-05T11:59:49.0172895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=warn ts=2026-05-05T11:59:52.165118929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T11:59:52.270583231Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T11:59:57.164216293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T11:59:57.265562188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T11:59:57.273555329Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:00:02.164387444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:00:06.462374605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:00:06.467249062Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:00:07.16496471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:00:12.164990584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:00:14.146357241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:00:14.154215539Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:00:17.144259022Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:00:17.144570208Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-05T12:00:17.144617531Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975535 n=1
level=debug ts=2026-05-05T12:00:17.14466477Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975535 n=1
level=debug ts=2026-05-05T12:00:17.144659749Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:00:17.164587881Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-05T12:00:17.16459866Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-05T12:00:17.164615354Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:00:17.1646266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:00:17.1647261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:00:17.164760443Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-05T12:00:17.16477313Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:00:17.164796727Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.763µs success=true
level=debug ts=2026-05-05T12:00:17.165105512Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-05T12:00:18.98777675Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T12:00:19.020613992Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:00:20.583956015Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=warn ts=2026-05-05T12:00:22.165069952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:00:22.864579074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:00:22.878294519Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:00:24.001157715Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-05T12:00:27.16501174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:00:27.419140624Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=debug ts=2026-05-05T12:00:27.41917381Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-05T12:00:30.260702584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:00:30.268943622Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:00:30.83631366Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=warn ts=2026-05-05T12:00:32.164483959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:00:32.715442932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:00:34.254347998Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-05T12:00:37.165073936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:00:37.671659575Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-05T12:00:41.089408361Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-05-05T12:00:42.16424473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:00:44.507612609Z caller=checkpoint.go:372 msg="writing series" size="639 kB"
level=info ts=2026-05-05T12:00:47.165008978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:00:47.165048096Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:00:47.16507796Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:00:47.202140166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:00:47.925560913Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-05T12:00:47.925591025Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-05T12:00:49.022538597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:00:51.343032012Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=warn ts=2026-05-05T12:00:52.16483666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:00:52.632533878Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:00:54.471415735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:00:54.482747776Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:00:54.760763668Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-05T12:00:57.164657567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:00:58.179270958Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-05T12:00:58.179297494Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=3
level=debug ts=2026-05-05T12:01:01.59634168Z caller=checkpoint.go:372 msg="writing series" size="518 kB"
level=warn ts=2026-05-05T12:01:02.164350692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:01:03.788019314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:01:03.792426835Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:01:05.014279277Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-05T12:01:06.900287085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:01:06.910435759Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:01:07.164773468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:01:08.431453163Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-05T12:01:09.885376623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:01:11.849613665Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-05T12:01:12.16476178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:01:13.779783554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:01:13.787609588Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:01:13.787636963Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:01:13.787681321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:01:13.795679696Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:01:15.267290624Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-05T12:01:17.143990078Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:01:17.144017875Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:01:17.164889174Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:01:17.164914114Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:01:17.164922507Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:01:17.165003298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:01:18.140125796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:01:18.149039193Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:01:18.684981984Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-05T12:01:18.685008668Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=6
ts=2026-05-05T12:01:19.025282334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:01:20.595594461Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:01:22.102435058Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-05T12:01:22.164196104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:01:25.604403227Z caller=checkpoint.go:372 msg="writing series" size="50 MB"
level=debug ts=2026-05-05T12:01:25.604442627Z caller=checkpoint.go:384 msg="flushing series" totalSize="50 MB" series=2
level=info ts=2026-05-05T12:01:26.49724082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:01:26.512715655Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:01:27.169376223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:01:28.937902171Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
ts=2026-05-05T12:01:29.109461216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35882"
level=warn ts=2026-05-05T12:01:32.164147254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:01:32.354974157Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-05T12:01:35.774353735Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-05T12:01:37.164238866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:01:39.190951361Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-05T12:01:42.164864469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:01:42.6086714Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-05T12:01:44.561404753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:01:44.573796755Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:01:46.026677988Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:01:46.026703365Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-05T12:01:47.164032809Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:01:47.164091891Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:01:47.16410136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:01:47.164172096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T12:01:48.147777136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48550"
ts=2026-05-05T12:01:49.02790114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:01:49.410221123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:01:49.421484311Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:01:49.443937133Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=debug ts=2026-05-05T12:01:49.580262661Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:01:52.16496785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:01:52.862245756Z caller=checkpoint.go:372 msg="writing series" size="388 kB"
ts=2026-05-05T12:01:56.107579597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:58914"
level=debug ts=2026-05-05T12:01:56.279440394Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-05T12:01:56.281174896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:01:56.290269872Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:01:57.164593658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T12:01:59.112333132Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56106"
level=debug ts=2026-05-05T12:01:59.314813451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:01:59.317194689Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T12:01:59.697949677Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-05T12:02:01.526753168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:02:01.530415916Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:02:01.530437774Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:02:01.530460564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:02:01.533713978Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:02:02.164830768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:03.115676661Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:02:03.115725101Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-05T12:02:04.455347205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:02:04.462975178Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:02:04.462997155Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:02:04.463024097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:02:04.470207805Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:02:06.541912425Z caller=checkpoint.go:372 msg="writing series" size="9.3 MB"
level=debug ts=2026-05-05T12:02:06.541946091Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.3 MB" series=1
level=warn ts=2026-05-05T12:02:07.164120425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:09.950542207Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-05T12:02:12.165236153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:13.368132616Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
level=debug ts=2026-05-05T12:02:16.785694136Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-05T12:02:16.899131268Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:02:17.144579578Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:02:17.145590098Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:02:17.165027558Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:02:17.165038355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:17.165082963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:02:17.165093704Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T12:02:18.149011892Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:55544"
level=info ts=2026-05-05T12:02:18.565780908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:02:18.574689266Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dlty\"}"
ts=2026-05-05T12:02:19.030735083Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:02:20.203493856Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=info ts=2026-05-05T12:02:21.436161954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:02:21.439751324Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:02:22.164902197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:23.621099085Z caller=checkpoint.go:372 msg="writing series" size="9.8 kB"
level=debug ts=2026-05-05T12:02:27.039077309Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-05T12:02:27.164955079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:30.457113153Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-05T12:02:30.482187289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:02:30.493301298Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:02:30.826409813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:02:30.833577508Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:02:30.83360147Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:02:30.833629264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:02:30.840287325Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:02:32.164272388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:33.87448755Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=warn ts=2026-05-05T12:02:37.1641789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:37.292736995Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-05T12:02:40.710530236Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T12:02:40.710559832Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-05T12:02:41.566715571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:02:41.577616834Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:02:41.580643448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:02:41.587404081Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:02:42.165214224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:02:43.97524738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:02:43.97875051Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:02:44.12820626Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=info ts=2026-05-05T12:02:47.164406656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:02:47.164408517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:47.164437635Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:02:47.164446498Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:02:47.545563141Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
ts=2026-05-05T12:02:49.032857566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:02:49.216983475Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:02:49.217050014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:02:49.221364615Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:02:49.961014977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:02:50.963341203Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=debug ts=2026-05-05T12:02:50.963371701Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-05T12:02:52.037223601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:02:52.04508224Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:02:52.164903147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:54.380951022Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
ts=2026-05-05T12:02:56.112857539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:38602"
level=warn ts=2026-05-05T12:02:57.164749602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:02:57.801288978Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-05T12:03:01.217017151Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-05T12:03:02.164861134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:03:03.39133526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:03:03.403774281Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:03:04.633536289Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-05T12:03:04.733792898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:03:04.738162558Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:03:07.164669819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:03:08.051414079Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-05T12:03:11.469531868Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T12:03:12.165031942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:03:14.887244414Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-05T12:03:16.33878563Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:03:17.144507995Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:03:17.145397092Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:03:17.164878287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:03:17.164903481Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:03:17.164923143Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:03:17.164941198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:03:18.305303801Z caller=checkpoint.go:372 msg="writing series" size="346 kB"
ts=2026-05-05T12:03:19.035522375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:03:21.722776557Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-05T12:03:22.164932632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:03:25.141434376Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:03:25.141455927Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=10
level=info ts=2026-05-05T12:03:25.82536491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:03:25.831594658Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:03:26.316946584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:03:26.325293533Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:03:27.164549385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:03:27.413902163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:03:27.41822169Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:03:28.558132309Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
ts=2026-05-05T12:03:29.12228558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56138"
level=info ts=2026-05-05T12:03:31.22827773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:03:31.23601543Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:03:31.975951803Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:03:32.164514005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:03:35.394095463Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-05T12:03:37.164203778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:03:38.142823105Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:03:38.810829923Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:03:41.999817359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:03:42.00795434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:03:42.164328517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:03:42.228730823Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=debug ts=2026-05-05T12:03:45.646900139Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-05T12:03:47.164408867Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:03:47.164449821Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:03:47.164444685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:03:47.164460733Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:03:48.339417924Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:03:48.33947848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="378 kB" avg_comp="378 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:03:48.343661668Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-05T12:03:49.0382824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:03:49.064530758Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:03:49.064556865Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-05T12:03:52.16440658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:03:52.48224586Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-05T12:03:55.909675603Z caller=checkpoint.go:372 msg="writing series" size="9.7 MB"
level=debug ts=2026-05-05T12:03:55.909700494Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.7 MB" series=2
ts=2026-05-05T12:03:56.117976847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:59340"
level=warn ts=2026-05-05T12:03:57.164189903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:03:59.317311876Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=info ts=2026-05-05T12:04:00.182104462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:04:00.188336521Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:04:02.165198659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:04:02.735723587Z caller=checkpoint.go:372 msg="writing series" size="747 kB"
level=info ts=2026-05-05T12:04:03.287000463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:04:03.294470474Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:04:06.155272436Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-05T12:04:06.155301861Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.3 MB" series=3
level=warn ts=2026-05-05T12:04:07.164441911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:04:07.917617131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:04:07.920748065Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:04:09.571034709Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-05T12:04:12.164617231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:04:12.988634478Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-05T12:04:14.335182649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:04:14.342496571Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:04:15.265609343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="401 MB" avg_uncomp="24 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:04:15.272835042Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:04:16.406832268Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-05T12:04:16.406855254Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-05T12:04:17.143989211Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:04:17.144604393Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:04:17.164031184Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:04:17.164070109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:04:17.164088339Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:04:17.16409858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:04:17.311766924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:04:17.316091324Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:04:18.934209532Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T12:04:19.040775983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:04:19.823907657Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=warn ts=2026-05-05T12:04:22.164222708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:04:23.241354339Z caller=checkpoint.go:372 msg="writing series" size="400 kB"
level=debug ts=2026-05-05T12:04:26.659047334Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-05T12:04:27.164515303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:04:30.077657903Z caller=checkpoint.go:372 msg="writing series" size="952 kB"
level=debug ts=2026-05-05T12:04:30.077680434Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-05T12:04:32.164805447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:04:33.495216648Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-05-05T12:04:36.912965642Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=debug ts=2026-05-05T12:04:36.933737609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:04:37.164932183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:04:38.474132578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:04:38.482504321Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:04:39.208427531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:04:39.211764419Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:04:40.330199284Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=warn ts=2026-05-05T12:04:42.164581214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:04:43.74768245Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-05T12:04:44.318325769Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:04:47.164518031Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:04:47.164522799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:04:47.164552736Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:04:47.164560444Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:04:47.165739127Z caller=checkpoint.go:384 msg="flushing series" totalSize="592 kB" series=4
level=info ts=2026-05-05T12:04:47.173437333Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-05T12:04:47.175511736Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010502866s
level=info ts=2026-05-05T12:04:49.023528805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:04:49.027271887Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T12:04:49.042707625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=warn ts=2026-05-05T12:04:52.164505861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:04:56.233875236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:04:56.241205275Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:04:57.164510069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:04:57.406280999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:04:57.413882616Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:05:01.330676426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:05:01.334507803Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:05:02.164265396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:05:04.584189196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:05:04.591099644Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:05:07.164330707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:05:12.164793608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:05:17.144477387Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:05:17.145244845Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:05:17.16467405Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T12:05:17.164706363Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:05:17.164725183Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:05:17.164732892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:05:17.164746783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:05:17.164839399Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-05T12:05:17.16484718Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:05:17.16486526Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.896µs success=true
level=debug ts=2026-05-05T12:05:17.165093906Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-05T12:05:18.163524077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:55672"
ts=2026-05-05T12:05:19.045422612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:05:19.397384157Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:05:20.58363208Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=warn ts=2026-05-05T12:05:22.165077044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:05:24.00124204Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-05T12:05:27.16416362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:05:27.418551323Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-05T12:05:27.418574086Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-05T12:05:29.553624551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:05:29.5578677Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:05:30.713538123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:05:30.721201157Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:05:30.836994883Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-05T12:05:32.164696664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:05:33.606234314Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:05:34.254084654Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-05T12:05:37.164822718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:05:37.672577234Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=info ts=2026-05-05T12:05:40.675190607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:05:40.682957432Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:05:41.09021408Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-05T12:05:42.1643981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:05:43.548416259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:05:43.55175918Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:05:44.507523756Z caller=checkpoint.go:372 msg="writing series" size="639 kB"
level=info ts=2026-05-05T12:05:45.15329269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:05:45.158164814Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:05:47.164796138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:05:47.164831072Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:05:47.164841799Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:05:47.164866691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:05:47.925290564Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-05T12:05:47.925316823Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-05T12:05:48.064895534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:05:48.072806987Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-05T12:05:49.048960964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:05:50.833276157Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:05:51.342270381Z caller=checkpoint.go:372 msg="writing series" size="638 B"
level=info ts=2026-05-05T12:05:51.898304987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:05:51.907832703Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:05:52.16512219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:05:54.760354621Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-05T12:05:57.16470409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:05:58.180643286Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-05T12:05:58.180667416Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=debug ts=2026-05-05T12:06:01.596397274Z caller=checkpoint.go:372 msg="writing series" size="398 kB"
level=warn ts=2026-05-05T12:06:02.164833497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:06:05.013984902Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=warn ts=2026-05-05T12:06:07.164203303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:06:08.431631121Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-05T12:06:10.158601867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:06:10.166479869Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:06:11.849861559Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-05T12:06:12.164831785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:06:15.266456695Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-05T12:06:17.143893049Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:06:17.144227694Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:06:17.164729907Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:06:17.16474483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:06:17.164758215Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:06:17.164769226Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:06:18.685885813Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-05T12:06:18.685907285Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=6
ts=2026-05-05T12:06:19.051311734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:06:20.152600966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:06:20.159221264Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:06:20.159243784Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:06:20.159270312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:06:20.165519933Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:06:22.102249632Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-05T12:06:22.164638343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:06:24.246624377Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:06:25.582981803Z caller=checkpoint.go:372 msg="writing series" size="53 MB"
level=debug ts=2026-05-05T12:06:25.583010633Z caller=checkpoint.go:384 msg="flushing series" totalSize="53 MB" series=2
ts=2026-05-05T12:06:26.130690065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:43370"
level=warn ts=2026-05-05T12:06:27.164982192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:06:28.112749065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:06:28.117148532Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:06:28.937824217Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-05T12:06:32.164978065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:06:32.354993249Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-05T12:06:32.912495633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:06:32.914343446Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T12:06:33.037932852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:06:33.042209772Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:06:35.773100482Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-05T12:06:37.164838678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:06:38.067186047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:06:38.075203917Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:06:39.190999561Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:06:39.380493585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:06:39.388008894Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:06:42.165139715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:06:42.321558241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:06:42.341477645Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:06:42.608447172Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:06:42.894339001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:06:42.905331785Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:06:46.027907681Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:06:46.027932744Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-05T12:06:47.164811357Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:06:47.164860479Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:06:47.164855017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:06:47.164871807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T12:06:48.171777708Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:55168"
ts=2026-05-05T12:06:49.054263806Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:06:49.444176661Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=warn ts=2026-05-05T12:06:52.164620447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:06:52.862632688Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=debug ts=2026-05-05T12:06:53.397458473Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T12:06:56.132306573Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:60524"
level=debug ts=2026-05-05T12:06:56.279600087Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-05T12:06:57.164198777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:06:58.004987389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:06:58.010254834Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:06:59.697262242Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-05T12:07:02.16482035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:03.115644659Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:07:03.115667224Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-05T12:07:06.547220815Z caller=checkpoint.go:372 msg="writing series" size="9.7 MB"
level=debug ts=2026-05-05T12:07:06.547253652Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.7 MB" series=1
level=warn ts=2026-05-05T12:07:07.164436976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:09.950253282Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-05T12:07:12.164206875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:13.36853734Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
level=info ts=2026-05-05T12:07:13.753715341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:07:13.760116046Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:07:16.78575497Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-05T12:07:17.143901426Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:07:17.144218978Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:07:17.164639052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:07:17.164663118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:07:17.164671436Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:07:17.164701062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:07:18.135661347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:07:18.143747426Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T12:07:19.056476826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:07:20.203391002Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=warn ts=2026-05-05T12:07:22.165004443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:23.6211012Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-05T12:07:23.729341352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:07:24.172864747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:07:24.180178294Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:07:26.129130942Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:07:27.038967679Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=warn ts=2026-05-05T12:07:27.164582257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:07:27.188725419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:07:27.192000159Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:07:29.132395405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:07:29.139541499Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:07:29.139565596Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:07:29.139605218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:07:29.146316655Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:07:30.457036844Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T12:07:32.164123548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:33.874115801Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-05T12:07:37.164544174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:37.292105757Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-05T12:07:40.541613156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:07:40.550767936Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:07:40.710265158Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T12:07:40.710289901Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=10
level=info ts=2026-05-05T12:07:41.731827403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:07:41.739913346Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:07:42.164620427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:44.12814529Z caller=checkpoint.go:372 msg="writing series" size="624 kB"
level=info ts=2026-05-05T12:07:47.164193301Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:07:47.164243279Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:07:47.164262236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:07:47.164269508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:47.546355674Z caller=checkpoint.go:372 msg="writing series" size="686 kB"
level=debug ts=2026-05-05T12:07:47.546377197Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
ts=2026-05-05T12:07:48.176387752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51540"
ts=2026-05-05T12:07:49.057815359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:07:50.96339051Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-05T12:07:52.164983966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:54.380310604Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-05T12:07:55.242886683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:07:55.248772022Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:07:55.514422498Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:07:57.164940791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:07:57.799067672Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-05T12:07:58.095707293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:07:58.100685224Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:07:59.150989807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:50762"
level=debug ts=2026-05-05T12:08:01.079010853Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:08:01.216544123Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-05T12:08:01.772390307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:08:01.778956943Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:08:02.164990904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:08:04.634088025Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-05T12:08:07.165089832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:08:08.051545892Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-05-05T12:08:11.469241693Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T12:08:11.57069315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:08:11.578314633Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:08:12.164333838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:08:14.887328996Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-05T12:08:17.144074517Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:08:17.144074743Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:08:17.164630045Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:08:17.164663731Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:08:17.164671565Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:08:17.164670549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:08:18.305428922Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-05T12:08:18.914028838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:08:18.920041375Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:08:18.920092868Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:08:18.920120434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:08:18.92304136Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T12:08:19.059168548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:08:19.865546411Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:08:21.723223971Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-05T12:08:22.164702096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:08:22.77608712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:08:22.783771263Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:08:22.783795229Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:08:22.783823064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:08:22.790716134Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:08:25.140838579Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:08:25.140859911Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=11
level=warn ts=2026-05-05T12:08:27.164867293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:08:27.480778673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:08:27.488849181Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:08:28.557881848Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-05T12:08:31.975869015Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:08:32.164890475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:08:35.393632979Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-05T12:08:36.590118021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:08:36.594214922Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:08:36.671461222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:08:36.67951607Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:08:37.164600208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:08:37.817456914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:08:37.821241735Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:08:38.81145111Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T12:08:42.164337431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:08:42.228863269Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=debug ts=2026-05-05T12:08:45.646759938Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-05-05T12:08:47.164901365Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:08:47.164949561Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:08:47.164961309Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:08:47.165017004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T12:08:48.180472752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49254"
ts=2026-05-05T12:08:49.061874603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:08:49.064631224Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:08:49.064650648Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-05T12:08:52.164139119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:08:52.481732164Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-05T12:08:52.548096155Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:08:53.74254485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:08:53.749396385Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:08:54.131594823Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:08:55.912237133Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:08:55.912261607Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
ts=2026-05-05T12:08:56.14185756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:46934"
level=info ts=2026-05-05T12:08:56.9673822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:08:56.970892289Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:08:56.970915279Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:08:56.970938538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:08:56.973814842Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:08:57.164475366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:08:58.853115177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:08:58.861689556Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T12:08:59.156754531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41116"
level=debug ts=2026-05-05T12:08:59.317299435Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-05T12:09:02.165174034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:09:02.735658496Z caller=checkpoint.go:372 msg="writing series" size="839 kB"
level=info ts=2026-05-05T12:09:05.155820805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:09:05.163898514Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:09:06.154813435Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-05T12:09:06.154837953Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=3
level=warn ts=2026-05-05T12:09:07.16434086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:09:09.57028283Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-05-05T12:09:11.836785177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:09:11.843831534Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:09:11.843856998Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:09:11.843885895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:09:11.850149889Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:09:12.164525069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:09:12.988178677Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-05T12:09:15.010755729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:09:15.014614261Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:09:15.711325758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:09:15.715190027Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:09:16.408323929Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-05T12:09:16.40835218Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-05-05T12:09:17.143677084Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:09:17.144763895Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:09:17.164338921Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:09:17.164363113Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:09:17.16437105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:09:17.164383377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T12:09:18.182249101Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:32860"
ts=2026-05-05T12:09:19.064795487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:09:19.824134602Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=debug ts=2026-05-05T12:09:21.569215877Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:09:22.165012588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:09:22.493284301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:09:22.502190657Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:09:23.241699457Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=debug ts=2026-05-05T12:09:26.659194371Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-05T12:09:27.16461829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:09:30.077850636Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:09:30.077876453Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-05T12:09:30.608640181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:09:30.612390849Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:09:32.164852626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:09:33.494259306Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-05-05T12:09:36.912972506Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=warn ts=2026-05-05T12:09:37.165072199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:09:40.330083537Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=warn ts=2026-05-05T12:09:42.164895882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:09:43.748383931Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-05-05T12:09:44.029930475Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:09:45.78145053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:09:47.164917516Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:09:47.16491856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:09:47.164950105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:09:47.164961959Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:09:47.165068221Z caller=checkpoint.go:384 msg="flushing series" totalSize="732 kB" series=4
level=info ts=2026-05-05T12:09:47.170913815Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-05T12:09:47.172576153Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007577057s
ts=2026-05-05T12:09:48.183915282Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39140"
level=info ts=2026-05-05T12:09:48.26957135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:09:48.276784986Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-05T12:09:49.067860346Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:09:50.596672582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:09:50.600514286Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:09:51.760931362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:09:51.76896663Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:09:52.164722809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T12:09:56.14692621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:33702"
level=info ts=2026-05-05T12:09:56.84865572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:09:56.856756011Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:09:57.165035357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:10:00.839800377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:10:00.847147606Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:10:02.165040788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:10:07.164718198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:10:07.413622768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:10:07.419202008Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:10:12.164484685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:10:17.144095264Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:10:17.144480876Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:10:17.163980545Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T12:10:17.164996288Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:10:17.165024526Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:10:17.16503322Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:10:17.165077639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:10:17.193395311Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-05T12:10:17.19342174Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:10:17.193457307Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=30.126µs success=true
level=debug ts=2026-05-05T12:10:17.193710514Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-05T12:10:19.070787455Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:10:19.357472712Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:10:19.606118241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:10:19.610954669Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:10:19.610977406Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:10:19.611001952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:10:19.613785761Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:10:19.786117615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:10:19.78776064Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T12:10:20.613027698Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=warn ts=2026-05-05T12:10:22.164646411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:10:24.0300035Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-05T12:10:27.165200714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:10:27.447458752Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-05T12:10:27.447490305Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-05T12:10:30.865089592Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-05T12:10:32.164931789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:10:34.283049609Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-05T12:10:35.112178789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:10:35.118656663Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:10:36.106009831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:10:36.112800818Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:10:37.164600276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:10:37.700613368Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=info ts=2026-05-05T12:10:38.833808721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:10:38.839551955Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:10:40.779292949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:10:40.788172541Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:10:41.118848795Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=warn ts=2026-05-05T12:10:42.164754142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:10:44.227929907Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:10:44.53628244Z caller=checkpoint.go:372 msg="writing series" size="752 kB"
level=debug ts=2026-05-05T12:10:44.536304789Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-05T12:10:46.676929268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:10:46.683686193Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:10:47.164568789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:10:47.164598118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:10:47.164609186Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:10:47.164627804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:10:47.953610693Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
ts=2026-05-05T12:10:49.072823049Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:10:51.371595422Z caller=checkpoint.go:372 msg="writing series" size="638 B"
level=warn ts=2026-05-05T12:10:52.164727822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:10:54.490487723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:10:54.499708632Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:10:54.789554367Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-05T12:10:56.731116253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:10:56.734262781Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:10:57.164590859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:10:58.062515212Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:10:58.208107136Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-05T12:10:58.208132669Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=4
ts=2026-05-05T12:10:59.168402922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42786"
level=info ts=2026-05-05T12:11:00.072207007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:11:00.075344571Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:11:01.625285451Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=warn ts=2026-05-05T12:11:02.164596115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:05.042244385Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=warn ts=2026-05-05T12:11:07.1650171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:08.460372409Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-05T12:11:11.87838313Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-05T12:11:11.886335776Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:11:12.164739228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:15.295828265Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-05T12:11:15.777690054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:11:15.784338851Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:11:17.144254186Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:11:17.144691716Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:11:17.164241621Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:11:17.164263378Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:11:17.164271721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:11:17.164303094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:18.715679298Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-05T12:11:18.715708443Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=6
ts=2026-05-05T12:11:19.074855614Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:11:22.131462375Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-05T12:11:22.16516942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:25.613402218Z caller=checkpoint.go:372 msg="writing series" size="54 MB"
level=debug ts=2026-05-05T12:11:25.613438045Z caller=checkpoint.go:384 msg="flushing series" totalSize="55 MB" series=2
level=warn ts=2026-05-05T12:11:27.164128716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:11:28.252754418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:11:28.261727835Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:11:28.967069234Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=info ts=2026-05-05T12:11:32.015048723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:11:32.023239496Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:11:32.164900596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:32.383598445Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-05T12:11:35.80137295Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-05T12:11:36.077175523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:11:36.084658738Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:11:37.164679445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:39.220003562Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-05T12:11:40.833579042Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:11:42.164652559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:42.637089961Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-05T12:11:46.056272666Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:11:46.05630053Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-05T12:11:46.836720616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:11:46.844667704Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:11:47.16490039Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:11:47.164896537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:47.164946143Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:11:47.164955723Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T12:11:48.410181969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:11:48.417367433Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T12:11:49.076477262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:11:49.473123581Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-05T12:11:49.672841167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:11:52.164400074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:52.89090458Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-05-05T12:11:54.779287884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:11:54.782568877Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:11:56.308736464Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-05T12:11:57.16460747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:11:59.725634359Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-05T12:12:02.165049859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:12:03.143625381Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:12:03.143649753Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=debug ts=2026-05-05T12:12:06.568210744Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-05T12:12:06.568233567Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=1
level=warn ts=2026-05-05T12:12:07.164190173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:12:09.979602318Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-05-05T12:12:12.1649338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:12:13.396673133Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
level=info ts=2026-05-05T12:12:13.400763791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:12:13.405522586Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:12:14.861543941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:12:14.869972828Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:12:15.778134227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:12:16.814974886Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=info ts=2026-05-05T12:12:17.144355636Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:12:17.14439523Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:12:17.164940367Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:12:17.16496663Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:12:17.164976887Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:12:17.165035733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T12:12:18.196308166Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59052"
ts=2026-05-05T12:12:19.078018404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:12:20.232724104Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=warn ts=2026-05-05T12:12:22.165087934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:12:23.649895326Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-05T12:12:23.792893075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:12:23.801676815Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-05T12:12:26.158203589Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:39392"
level=info ts=2026-05-05T12:12:26.545433131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="54 MB" avg_comp="2.9 MB" total_uncomp="449 MB" avg_uncomp="24 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:12:26.553241276Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:12:27.068511652Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-05T12:12:27.164440505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T12:12:29.176908715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51252"
level=debug ts=2026-05-05T12:12:30.48574226Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T12:12:32.164208855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:12:33.903817033Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
level=info ts=2026-05-05T12:12:36.599546617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:12:36.607037454Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:12:37.164555118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:12:37.321168127Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-05T12:12:37.321194274Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-05T12:12:40.739412881Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=warn ts=2026-05-05T12:12:42.164858651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:12:43.975796142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:12:43.979588854Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:12:44.156205703Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=info ts=2026-05-05T12:12:47.164177173Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:12:47.164205531Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:12:47.164194416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:12:47.164214253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:12:47.261952059Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:12:47.574287377Z caller=checkpoint.go:372 msg="writing series" size="803 kB"
level=debug ts=2026-05-05T12:12:47.574316522Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
ts=2026-05-05T12:12:48.197384536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:46930"
ts=2026-05-05T12:12:49.080589782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:12:50.2997352Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:12:50.991925702Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-05T12:12:52.164722156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:12:54.409445152Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-05T12:12:54.417529488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:12:54.424693931Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:12:57.164122348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:12:57.827683391Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
ts=2026-05-05T12:12:59.18004583Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54380"
level=info ts=2026-05-05T12:13:00.92311763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:13:00.931726717Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:13:01.24521633Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-05T12:13:02.165093649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:04.662523825Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=info ts=2026-05-05T12:13:04.762774083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:13:04.770558183Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:13:07.164414505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:13:07.888315246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:13:07.892424544Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:13:08.080004818Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-05T12:13:09.155343407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="473 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:13:09.161520258Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:13:11.497672379Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T12:13:12.164144954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:14.915796957Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=debug ts=2026-05-05T12:13:15.851564276Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:13:17.143463677Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:13:17.144560874Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:13:17.164992377Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:13:17.16501883Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:13:17.165029433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:13:17.165048466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:18.333259228Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
ts=2026-05-05T12:13:19.082841342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:13:21.505211027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:13:21.514573926Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:13:21.514607984Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:13:21.514639386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:13:21.52324329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:13:21.751094545Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-05T12:13:22.165090138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:23.524688768Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:13:25.170010948Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:13:25.170036741Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=11
level=warn ts=2026-05-05T12:13:27.164107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:28.586382812Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-05T12:13:32.003893782Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:13:32.165044997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:13:33.932608822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:13:33.942445564Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:13:35.162786301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:13:35.166527139Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:13:35.42221692Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-05T12:13:37.164292202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:38.839873235Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-05T12:13:41.418095372Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:13:42.164498533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:42.258182573Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=debug ts=2026-05-05T12:13:45.675756827Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-05T12:13:47.16487006Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:13:47.164872206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:47.164898546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:13:47.164906646Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-05T12:13:49.085725175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:13:49.093489254Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:13:49.093509496Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-05T12:13:50.218685382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:13:50.226637707Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:13:52.164484003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:52.510835266Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-05T12:13:54.211822392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:13:54.222189713Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:13:55.946482941Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:13:55.946505851Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
ts=2026-05-05T12:13:56.166323142Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:54204"
level=warn ts=2026-05-05T12:13:57.164936812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:13:59.346507869Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-05-05T12:14:00.8805195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:14:00.88807691Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:14:02.164184364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:14:02.764635294Z caller=checkpoint.go:372 msg="writing series" size="801 kB"
level=debug ts=2026-05-05T12:14:04.229011843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:14:04.231213982Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T12:14:04.419723444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:14:04.428696144Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:14:04.572256705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:14:04.575742493Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:14:06.054369132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="473 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:14:06.062498801Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:14:06.062522874Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:14:06.062563946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="473 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:14:06.070254022Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:14:06.183939923Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-05T12:14:06.183966603Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=3
level=warn ts=2026-05-05T12:14:07.164182634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:14:09.599224084Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-05T12:14:12.164402287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:14:13.017003576Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=debug ts=2026-05-05T12:14:13.778342388Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:14:16.436468072Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-05T12:14:16.436489447Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=info ts=2026-05-05T12:14:17.144457401Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:14:17.144880402Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:14:17.164237314Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:14:17.164262371Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:14:17.164272321Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:14:17.164261655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T12:14:19.089006147Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:14:19.852705756Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=warn ts=2026-05-05T12:14:22.16467625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:14:23.270808359Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-05T12:14:23.753906825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="473 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:14:23.760947992Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:14:25.479821129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:14:25.483022169Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T12:14:26.168484947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:43766"
level=debug ts=2026-05-05T12:14:26.688244406Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-05T12:14:27.164392544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-05T12:14:29.189298Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34674"
level=debug ts=2026-05-05T12:14:30.106139223Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-05T12:14:30.106160257Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=4
level=warn ts=2026-05-05T12:14:32.164871476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:14:33.523333462Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-05T12:14:36.267515676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:14:36.274890781Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:14:36.940837477Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=warn ts=2026-05-05T12:14:37.165090945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:14:40.225605741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:14:40.359328374Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-05T12:14:42.164835716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:14:43.776993027Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-05T12:14:43.947574004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.9 MB" total_uncomp="473 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:14:43.953720372Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:14:44.70896044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:14:44.716299047Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:14:45.019561299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:14:45.023304739Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:14:45.558624035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:14:45.591710316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:14:45.600044065Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:14:45.600084127Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:14:45.60011081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:14:45.606844575Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:14:47.164720706Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:14:47.164748258Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:14:47.164756191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:14:47.164755806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:14:47.193851526Z caller=checkpoint.go:384 msg="flushing series" totalSize="541 kB" series=4
level=info ts=2026-05-05T12:14:47.197661063Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-05T12:14:47.199290797Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005656619s
ts=2026-05-05T12:14:48.207798447Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:44564"
ts=2026-05-05T12:14:49.091508295Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:14:51.642313445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:14:51.652420596Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:14:52.164723772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:14:57.164902767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:15:01.845958706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:15:01.857822547Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:15:02.165139219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:15:07.164598507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:15:12.164419016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:15:13.220885018Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:15:17.144202797Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:15:17.145348078Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-05T12:15:17.145413978Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975536 n=1
level=debug ts=2026-05-05T12:15:17.145474934Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975536 n=1
level=debug ts=2026-05-05T12:15:17.145694299Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:15:17.164009144Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T12:15:17.164027468Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:15:17.164072087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:15:17.164082733Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:15:17.164119173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:15:17.164193695Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-05T12:15:17.164210817Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:15:17.164238768Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=20.356µs success=true
level=debug ts=2026-05-05T12:15:17.164529652Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-05T12:15:17.637010337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:15:17.641208354Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:15:17.641242236Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:15:17.641279324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:15:17.644464787Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T12:15:19.096100917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:15:20.499179046Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=debug ts=2026-05-05T12:15:20.516292336Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:15:22.164453163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:15:23.831741296Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-05T12:15:27.164864204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:15:27.165022763Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-05-05T12:15:27.165038493Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-05T12:15:28.020662166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:15:28.026578072Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:15:28.670642244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:15:28.678167693Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:15:30.498514718Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-05T12:15:32.164325151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:15:33.068876814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:15:33.073410221Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:15:33.073430017Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:15:33.073453557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:15:33.076647705Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:15:33.831891068Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-05T12:15:33.993264583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:15:34.000306408Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:15:34.085522086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:15:34.088695632Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:15:37.059834209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:15:37.066915536Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:15:37.066936392Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:15:37.066960084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:15:37.073495208Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:15:37.164681078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:15:37.164889362Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-05T12:15:40.498682327Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-05T12:15:42.164835118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:15:43.272747709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:15:43.281158718Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:15:43.832109899Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=debug ts=2026-05-05T12:15:43.832139932Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-05-05T12:15:44.074579626Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:15:44.632275411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:15:44.636614177Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:15:47.164293717Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:15:47.164341412Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:15:47.16435175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:15:47.164341457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:15:47.165526407Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
ts=2026-05-05T12:15:48.211583416Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36064"
ts=2026-05-05T12:15:49.100493987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:15:49.765322106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:15:49.773187218Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:15:50.49837349Z caller=checkpoint.go:372 msg="writing series" size="627 B"
level=info ts=2026-05-05T12:15:51.879685059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:15:51.882833147Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:15:52.164576269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:15:53.832184615Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-05T12:15:57.164104095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:15:57.168721154Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-05T12:15:57.168747901Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
ts=2026-05-05T12:15:59.198805376Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48092"
level=debug ts=2026-05-05T12:16:00.49898961Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-05T12:16:02.16507117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:16:03.831456159Z caller=checkpoint.go:372 msg="writing series" size="442 kB"
level=warn ts=2026-05-05T12:16:07.164731891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:16:07.164776342Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=debug ts=2026-05-05T12:16:10.497932278Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-05T12:16:12.164573271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:16:13.832129918Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-05T12:16:14.323270834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:16:14.326723976Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:16:14.47858208Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:16:14.703393162Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:16:17.143914767Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:16:17.144131872Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:16:17.164581804Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:16:17.164600624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:16:17.164632824Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:16:17.164643486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:16:17.164653933Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
level=info ts=2026-05-05T12:16:18.926686741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:16:18.937514875Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-05T12:16:19.103204508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:16:19.61489726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:16:19.618138651Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:16:19.819695509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:16:19.825797483Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:16:20.499986433Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T12:16:20.500010698Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=7
level=warn ts=2026-05-05T12:16:22.164891316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:16:22.642263139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:16:22.649048459Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:16:23.831238697Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
ts=2026-05-05T12:16:26.182414612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:36396"
level=warn ts=2026-05-05T12:16:27.16412697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:16:27.243537059Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-05T12:16:27.243576035Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=2
ts=2026-05-05T12:16:29.201872405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34214"
level=info ts=2026-05-05T12:16:30.932483787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:16:30.939602186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:16:32.164208301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:16:37.16454092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:16:37.858105463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:16:37.862628252Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:16:42.164707082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:16:47.164234331Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:16:47.164263199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:16:47.164272231Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:16:47.164266283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:16:47.93837509Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=debug ts=2026-05-05T12:16:48.366981482Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T12:16:49.10504507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:16:50.498290532Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-05T12:16:52.164345589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:16:53.036858541Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:16:53.832140484Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T12:16:57.165145229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:16:57.165220909Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
ts=2026-05-05T12:16:59.204611112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49616"
level=debug ts=2026-05-05T12:17:00.498538934Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-05T12:17:02.164768578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:03.831678032Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=info ts=2026-05-05T12:17:05.079119635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:17:05.088702282Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:17:06.722900066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:17:06.726782794Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:17:06.726802528Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:17:06.7268246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:17:06.731577984Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:17:06.837702998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:17:06.84523885Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:17:07.04180853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:17:07.045994911Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:17:07.164811976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:07.16553087Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:17:07.165546973Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-05T12:17:08.350332297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:17:08.35865395Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:17:10.498429732Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-05-05T12:17:12.164898002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:13.832636919Z caller=checkpoint.go:372 msg="writing series" size="558 kB"
level=debug ts=2026-05-05T12:17:16.391628611Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:17:17.143941152Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:17:17.144116826Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:17:17.164535263Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:17:17.164557203Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:17:17.164565782Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:17:17.164558653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:17.164759887Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
ts=2026-05-05T12:17:19.107491641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:17:19.909273395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:17:19.91370151Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:17:20.498902769Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-05T12:17:22.165156316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:17:22.214261836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:17:22.217705602Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:17:23.013600343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:17:23.020842052Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:17:23.832499233Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:17:23.832522513Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=warn ts=2026-05-05T12:17:27.165078814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:27.173955828Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-05T12:17:27.17397864Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=1
level=debug ts=2026-05-05T12:17:30.498737657Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-05-05T12:17:32.164320698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:33.832021956Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
level=warn ts=2026-05-05T12:17:37.165124558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:37.165139483Z caller=checkpoint.go:372 msg="writing series" size="2.1 kB"
level=debug ts=2026-05-05T12:17:40.498250427Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=debug ts=2026-05-05T12:17:41.32666873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:17:41.328470713Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T12:17:41.359008556Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:17:42.164264538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:43.831417317Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:17:47.164254681Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:17:47.16428369Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:17:47.164292346Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:17:47.164314165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:47.165441845Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
ts=2026-05-05T12:17:49.110725491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:17:49.78826719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:17:49.79322852Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:17:50.498242006Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-05T12:17:52.164698832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:53.831766014Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-05T12:17:55.98168248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:17:55.988788999Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-05T12:17:56.189581859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:41330"
level=warn ts=2026-05-05T12:17:57.165179982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:17:57.165422817Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-05T12:17:58.143861978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:17:58.154096502Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
ts=2026-05-05T12:17:59.210803079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:44932"
level=info ts=2026-05-05T12:17:59.497933315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:17:59.50489597Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:18:00.458726833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:18:00.462364526Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:18:00.498765631Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T12:18:00.49878246Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-05-05T12:18:02.16421279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:03.831730545Z caller=checkpoint.go:372 msg="writing series" size="542 kB"
level=info ts=2026-05-05T12:18:04.985498809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:18:04.988624262Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:18:06.960732614Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:18:07.164387053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:07.165745662Z caller=checkpoint.go:372 msg="writing series" size="728 kB"
level=debug ts=2026-05-05T12:18:07.165761891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-05T12:18:08.276746004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:18:08.284388107Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:18:10.49867239Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-05-05T12:18:12.165114266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:13.831503628Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-05T12:18:17.143891394Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:18:17.144615755Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:18:17.165029394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:18:17.165030501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:17.165063947Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:18:17.165072784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:18:17.165235017Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
ts=2026-05-05T12:18:19.112651409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:18:20.498409206Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-05T12:18:22.164398727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:23.831314329Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
ts=2026-05-05T12:18:26.192210392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:50150"
level=warn ts=2026-05-05T12:18:27.164399257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:27.165521989Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
ts=2026-05-05T12:18:29.214212406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:39862"
level=debug ts=2026-05-05T12:18:30.166079905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:18:30.498048539Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T12:18:32.164748496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:33.832176927Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-05T12:18:36.580286879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:18:36.589754409Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:18:37.165196818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:37.165588635Z caller=checkpoint.go:372 msg="writing series" size="459 kB"
level=debug ts=2026-05-05T12:18:37.165603514Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-05-05T12:18:40.497853151Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T12:18:41.832959434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:18:41.840992114Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:18:42.164891741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:42.87921757Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:18:42.8792388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="551 kB" avg_comp="551 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:18:42.883246143Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:18:43.832764295Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:18:43.832785158Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-05T12:18:44.926746195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:18:44.935685715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:18:47.164541522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:18:47.164572517Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:18:47.164582798Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:18:47.164585107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:47.164672873Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-05T12:18:49.09802484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:18:49.101990614Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:18:49.115226465Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:18:50.498191879Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:18:52.164269401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:18:53.245236612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:18:53.252655958Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:18:53.831367894Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-05T12:18:56.194945869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:44468"
level=info ts=2026-05-05T12:18:56.386116383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:18:56.394190421Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:18:57.016127412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:18:57.019870071Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:18:57.164258209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:18:57.165352413Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-05T12:19:00.498455845Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-05-05T12:19:01.769635648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:19:01.774642244Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:19:02.164780335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:03.83136762Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-05T12:19:07.16485634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:07.164983922Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:19:07.165000102Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-05T12:19:09.363752492Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:19:10.498013407Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-05T12:19:12.164998568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:13.841352821Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:19:13.841375325Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=debug ts=2026-05-05T12:19:17.143864551Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:19:17.143894102Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:19:17.164263589Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:19:17.164271968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:17.164283771Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:19:17.164293131Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:19:17.165518951Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=debug ts=2026-05-05T12:19:17.334126827Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:19:19.117585251Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:19:20.49933712Z caller=checkpoint.go:372 msg="writing series" size="851 kB"
level=warn ts=2026-05-05T12:19:22.165079066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:23.835313414Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-05T12:19:23.835335355Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.8 MB" series=3
level=info ts=2026-05-05T12:19:26.433029651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:19:26.437414583Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:19:27.023177238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:19:27.031371349Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:19:27.164902572Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-05T12:19:27.164957554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:30.498418399Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:19:31.046260967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:19:31.050579595Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:19:32.165084196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:33.833583414Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-05T12:19:33.833607212Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=debug ts=2026-05-05T12:19:36.114220681Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:19:36.785280376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:19:36.789245773Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:19:37.164334771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:37.165599478Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=debug ts=2026-05-05T12:19:40.499026923Z caller=checkpoint.go:372 msg="writing series" size="342 kB"
level=warn ts=2026-05-05T12:19:42.164821369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:19:42.351470674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:19:42.359642464Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:19:43.831170784Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-05T12:19:47.164697633Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:19:47.164735259Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:19:47.164746997Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:19:47.164766066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:47.165481003Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:19:47.165497451Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=4
ts=2026-05-05T12:19:48.233293826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:55028"
ts=2026-05-05T12:19:49.120379158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:19:50.498201914Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-05T12:19:51.938685832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:19:51.963375115Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:19:52.165163914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:53.831963264Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-05T12:19:54.919925969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:19:54.928001773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:19:54.928026177Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:19:54.928072924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:19:54.934856241Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T12:19:56.199144672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:37666"
level=warn ts=2026-05-05T12:19:57.165036343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:19:57.165089857Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=debug ts=2026-05-05T12:20:00.498385009Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-05T12:20:01.508117439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:20:01.513319113Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:20:01.779468071Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:20:02.164272844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:20:03.83170266Z caller=checkpoint.go:384 msg="flushing series" totalSize="618 kB" series=4
level=info ts=2026-05-05T12:20:03.839529146Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-05T12:20:03.843047402Z caller=checkpoint.go:670 msg="checkpoint done" time=4m46.678605422s
level=warn ts=2026-05-05T12:20:07.164727481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:20:12.143123818Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:20:12.165087101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:20:12.85835131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:20:12.866979108Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:20:15.931909763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:20:15.937270965Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:20:17.143705067Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:20:17.144754611Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:20:17.164036084Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:20:17.164075652Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:20:17.164078382Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T12:20:17.164094697Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:20:17.164112205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:20:17.164243733Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-05T12:20:17.164252595Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:20:17.164270407Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.504µs success=true
level=debug ts=2026-05-05T12:20:17.164481197Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-05-05T12:20:18.234776003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:37944"
ts=2026-05-05T12:20:19.123792662Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:20:20.498896466Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=warn ts=2026-05-05T12:20:22.164413953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:20:23.587746654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:20:23.592115687Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:20:23.831296468Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-05T12:20:24.563346858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:20:24.571722136Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-05T12:20:26.20089993Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:57720"
level=warn ts=2026-05-05T12:20:27.164210881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:20:27.165376101Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=debug ts=2026-05-05T12:20:27.165390932Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-05T12:20:29.2250345Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45912"
level=debug ts=2026-05-05T12:20:30.499045258Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=info ts=2026-05-05T12:20:31.591729895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:20:31.608277818Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:20:31.608309019Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:20:31.608355566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:20:31.615940229Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:20:32.165119897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:20:32.213409659Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:20:32.213469406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="446 kB" avg_comp="446 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:20:32.217241972Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:20:33.831926299Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-05T12:20:36.316194854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:20:36.320753298Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:20:37.164133629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:20:37.165493726Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=debug ts=2026-05-05T12:20:40.498820114Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-05T12:20:42.165120093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:20:43.831805392Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
level=debug ts=2026-05-05T12:20:43.831829293Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T12:20:45.603630619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:20:45.611781603Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:20:46.245771009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:20:46.252936328Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:20:46.715268108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:20:46.723602179Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:20:47.089076936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:20:47.093198071Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:20:47.164568209Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:20:47.164594778Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:20:47.164604035Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:20:47.164628885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:20:47.164782564Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-05T12:20:48.23576639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:44332"
ts=2026-05-05T12:20:49.127136173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:20:50.498560937Z caller=checkpoint.go:372 msg="writing series" size="638 B"
level=warn ts=2026-05-05T12:20:52.16456286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:20:53.831679425Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-05T12:20:57.164716168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:20:57.167034592Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-05T12:20:57.16707709Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=4
level=debug ts=2026-05-05T12:21:00.498020358Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-05T12:21:00.701821535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:21:00.708202346Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:21:02.164867138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:21:03.669920872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:21:03.674623261Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:21:03.832511527Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=debug ts=2026-05-05T12:21:05.218388741Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:21:06.922664597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:21:06.926967834Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:21:07.164666997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:21:07.164784169Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=debug ts=2026-05-05T12:21:10.498584939Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-05T12:21:11.188245091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:21:12.164698399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:21:13.831946136Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-05T12:21:13.831967647Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T12:21:17.143880914Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:21:17.144139963Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:21:17.164515323Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:21:17.164545046Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:21:17.164556562Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:21:17.164615209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:21:17.164621429Z caller=checkpoint.go:372 msg="writing series" size="7.1 kB"
ts=2026-05-05T12:21:18.237911738Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51036"
ts=2026-05-05T12:21:19.129012086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:21:20.403963372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:21:20.413969947Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:21:20.500854598Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-05T12:21:20.500878558Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
level=warn ts=2026-05-05T12:21:22.16474767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:21:22.998322069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:21:23.005374322Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:21:23.831889637Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T12:21:24.37152711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:21:24.374937174Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:21:25.709895575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:21:25.715028277Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:21:27.164422831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:21:27.241997892Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-05T12:21:27.242035365Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=2
level=debug ts=2026-05-05T12:21:30.498357665Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-05T12:21:32.164655374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:21:33.38433001Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:21:33.831660746Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=info ts=2026-05-05T12:21:34.216597313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:21:34.225877224Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:21:34.225904581Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:21:34.225936757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:21:34.233656062Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:21:37.164830765Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T12:21:37.164840248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:21:40.39429345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:21:40.403010834Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:21:40.498409647Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-05T12:21:42.164969528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:21:43.831673753Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T12:21:47.164507075Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:21:47.164540251Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=debug ts=2026-05-05T12:21:47.164546449Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:21:47.164527982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:21:47.164558568Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T12:21:48.654312713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:21:48.662587467Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
ts=2026-05-05T12:21:49.130150338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:21:50.499658668Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:21:50.499691345Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-05T12:21:51.599872275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:21:51.60534978Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:21:51.605377178Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:21:51.605408045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:21:51.609143642Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:21:52.164399098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:21:52.840622636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:21:52.844795108Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:21:53.831704038Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=warn ts=2026-05-05T12:21:57.164104665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:21:57.16573994Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:22:00.195552738Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:22:00.235719344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:22:00.242044846Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:22:00.242081203Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:22:00.242109121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:22:00.244714521Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:22:00.498844257Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-05T12:22:02.164872105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:22:03.831304431Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-05T12:22:06.850001495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:22:06.854138725Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:22:07.164527229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:22:07.164988291Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:22:07.165004687Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-05T12:22:09.251343638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:22:09.25952033Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:22:10.51157268Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-05T12:22:10.511601043Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=info ts=2026-05-05T12:22:10.718327241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:22:10.721690568Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:22:11.584755795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:22:11.588268837Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T12:22:12.164789242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:22:12.711091531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:22:12.713339681Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T12:22:12.997226577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:22:13.001202191Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:22:13.832161981Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-05T12:22:16.9911151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:22:16.99990905Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:22:16.999948883Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:22:16.999997068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:22:17.007314725Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:22:17.14402969Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:22:17.144041175Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:22:17.164379053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:22:17.164389992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:22:17.164402016Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:22:17.164410144Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:22:17.164465133Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
ts=2026-05-05T12:22:19.132637385Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:22:20.498349383Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:22:22.164371041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:22:22.65621089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:22:22.665294248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:22:23.831644622Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-05T12:22:24.50619236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:22:24.515011179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T12:22:26.210049292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34954"
level=warn ts=2026-05-05T12:22:27.164365645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:22:27.16536973Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T12:22:29.985435194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:22:29.992799335Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:22:30.498742579Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-05T12:22:32.16511288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:22:33.384296077Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:22:33.831600726Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-05T12:22:35.449212411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:22:35.455454815Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:22:37.164691879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:22:37.16482658Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=debug ts=2026-05-05T12:22:40.498778499Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-05T12:22:42.164486982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:22:43.107294052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:22:43.111144939Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:22:43.831707193Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T12:22:43.831736148Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-05T12:22:45.74846159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:22:45.751629413Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:22:47.164352492Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:22:47.164386472Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:22:47.164397919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:22:47.164419723Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:22:47.164654279Z caller=checkpoint.go:372 msg="writing series" size="555 kB"
level=debug ts=2026-05-05T12:22:47.518605503Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:22:48.244789741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60312"
ts=2026-05-05T12:22:49.135648564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:22:50.498723087Z caller=checkpoint.go:372 msg="writing series" size="909 kB"
level=debug ts=2026-05-05T12:22:50.498755327Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=warn ts=2026-05-05T12:22:52.164339864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:22:53.832104542Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-05T12:22:55.7226875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:22:55.731049276Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:22:57.164175334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:22:57.165299748Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-05T12:22:57.322631196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:22:57.330446494Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:23:00.498591738Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=warn ts=2026-05-05T12:23:02.164689166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:23:02.313093948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:23:02.317221169Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:23:03.513864612Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:23:03.831857418Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-05T12:23:06.358108923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:23:06.364087763Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:23:07.164345142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:23:07.1655515Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-05-05T12:23:08.213359725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:23:08.22109536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:23:10.498870902Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-05T12:23:12.165121626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:23:13.831429616Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T12:23:17.143920542Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:23:17.144574907Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:23:17.164944784Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:23:17.164971058Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:23:17.164980223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:23:17.164979108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:23:17.165152126Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
ts=2026-05-05T12:23:19.138136241Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:23:20.498414552Z caller=checkpoint.go:372 msg="writing series" size="347 kB"
level=debug ts=2026-05-05T12:23:20.498441658Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-05T12:23:20.925561853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:23:20.93581473Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:23:21.976753664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:23:21.980705645Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:23:22.164690039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:23:23.831571815Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-05T12:23:25.109242748Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:23:26.21529918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:33202"
level=warn ts=2026-05-05T12:23:27.164967988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:23:27.167188034Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:23:27.167209863Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
ts=2026-05-05T12:23:29.241531904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:33282"
level=debug ts=2026-05-05T12:23:30.498370528Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=debug ts=2026-05-05T12:23:30.924700291Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:23:32.16471248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:23:33.83124777Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:23:37.164856743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:23:37.165014499Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-05T12:23:40.498614781Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:23:41.316186435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:23:41.32316034Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:23:41.573275853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:23:41.577220453Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:23:42.16447598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:23:43.831949974Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-05-05T12:23:47.164643905Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:23:47.164685973Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:23:47.164698202Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:23:47.164735318Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-05T12:23:47.164732355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:23:47.902874062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:23:47.906572171Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:23:47.906603669Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:23:47.906639936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:23:47.910250742Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-05T12:23:49.140237915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:23:50.498490955Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:23:50.498517479Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-05T12:23:50.519438269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:23:50.526729426Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:23:52.164661383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:23:52.847303082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:23:52.854404769Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:23:53.83121208Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-05-05T12:23:56.858777427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:23:56.866467713Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:23:57.164546837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:23:57.178435677Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:23:57.178462658Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=info ts=2026-05-05T12:23:57.799146453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:23:57.806921261Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:23:59.830764532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:23:59.834994422Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:24:00.498559358Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-05T12:24:02.164656537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:24:03.832037099Z caller=checkpoint.go:372 msg="writing series" size="902 kB"
level=debug ts=2026-05-05T12:24:03.83208027Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-05-05T12:24:04.184070546Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:24:05.477048181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:24:07.164715287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:24:07.167585197Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-05T12:24:07.167608872Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=1
level=debug ts=2026-05-05T12:24:10.497914183Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-05-05T12:24:11.804649217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:24:11.809360437Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:24:12.164956469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:24:13.831255067Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:24:17.143906162Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:24:17.143917268Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:24:17.164343886Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:24:17.164343102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:24:17.164370402Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:24:17.164381791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:24:17.165807726Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-05T12:24:17.165824625Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=3
level=info ts=2026-05-05T12:24:17.971525512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:24:17.979231716Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-05T12:24:19.141922187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:24:20.498597002Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=warn ts=2026-05-05T12:24:22.165040045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:24:23.83171968Z caller=checkpoint.go:372 msg="writing series" size="502 kB"
level=warn ts=2026-05-05T12:24:27.164197101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:24:27.171170854Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-05T12:24:27.784043915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:24:27.787956338Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:24:30.499636646Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:24:30.499662373Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=4
level=warn ts=2026-05-05T12:24:32.164290482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:24:33.584985213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:24:33.588808789Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:24:33.592925517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:24:33.601643783Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:24:33.831318579Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=debug ts=2026-05-05T12:24:33.908597269Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:24:36.552064598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:24:36.560115722Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:24:36.83580862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:24:36.839124022Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:24:37.164768037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:24:37.164977121Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-05T12:24:40.499898509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:24:40.500236583Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=error ts=2026-05-05T12:24:40.509265102Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:24:42.165157146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:24:43.493684142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:24:43.497596386Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:24:43.831613856Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=debug ts=2026-05-05T12:24:47.164623682Z caller=checkpoint.go:384 msg="flushing series" totalSize="670 kB" series=4
level=info ts=2026-05-05T12:24:47.164673212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:24:47.164708355Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:24:47.164719006Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:24:47.164725888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:24:47.174496505Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-05T12:24:47.177625409Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.013209361s
ts=2026-05-05T12:24:49.144622642Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=warn ts=2026-05-05T12:24:52.164693888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:24:57.164405003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:24:59.698428138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:24:59.974306278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:24:59.981890128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:25:02.164928025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:25:03.63016209Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:25:05.870974052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:25:05.874844896Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:25:05.874866099Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:25:05.874893155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:25:05.877756028Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:25:07.16504833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:25:12.164923468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:25:15.333175183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:25:15.338016345Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:25:16.706631562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:25:16.710764876Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:25:17.143689617Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:25:17.144725697Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:25:17.164018988Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T12:25:17.164031775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:25:17.164093402Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:25:17.164106563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:25:17.164151246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:25:17.164320933Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-05T12:25:17.164330573Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:25:17.164350319Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.63µs success=true
level=debug ts=2026-05-05T12:25:17.164587576Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-05T12:25:18.804364105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:25:18.808642078Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-05T12:25:19.148004302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:25:20.499858946Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=info ts=2026-05-05T12:25:21.430355646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:25:21.437017719Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:25:22.164609337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:25:23.83165496Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-05T12:25:25.879751348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:25:25.883312037Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:25:27.164199373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:25:27.165383304Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-05-05T12:25:27.165404061Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-05T12:25:29.25386395Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57732"
level=info ts=2026-05-05T12:25:30.154097227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:25:30.161950149Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:25:30.201042481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:25:30.207787734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:25:30.498531415Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-05T12:25:32.164268199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:25:33.831990578Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-05T12:25:37.164084155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:25:37.165196505Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-05-05T12:25:37.517432353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:25:37.520984387Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:25:39.249592611Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:25:40.498411917Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-05T12:25:42.164946104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:25:43.832700704Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
level=info ts=2026-05-05T12:25:47.164197571Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:25:47.164195374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:25:47.164229286Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:25:47.164246829Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:25:47.165460686Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:25:47.165474792Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-05T12:25:48.259547712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35240"
ts=2026-05-05T12:25:49.150205807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:25:50.498771338Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-05-05T12:25:52.165239361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:25:53.832133266Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-05T12:25:54.674669394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:25:54.685363567Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:25:57.164099698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:25:57.167040436Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-05T12:25:57.167090874Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=info ts=2026-05-05T12:25:57.911318016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:25:57.91467954Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:25:58.855430779Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:25:58.857149422Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T12:26:00.498156107Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-05T12:26:02.165108565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:26:03.831691138Z caller=checkpoint.go:372 msg="writing series" size="602 kB"
level=info ts=2026-05-05T12:26:06.376932782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:26:06.380523044Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:26:07.164852888Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=warn ts=2026-05-05T12:26:07.16484975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:26:07.677525807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:26:07.681021884Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:26:10.498237661Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-05T12:26:12.165111693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:26:13.148390963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:26:13.15649665Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:26:13.156521264Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:26:13.15654976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:26:13.16351995Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:26:13.323828271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:26:13.327274436Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:26:13.543311811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:26:13.543309731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="29 kB" avg_uncomp="29 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:26:13.546398411Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:26:13.832264468Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-05T12:26:13.832308352Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-05T12:26:13.844192317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="194 kB" avg_uncomp="194 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:26:13.84767907Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:26:14.143628426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="56 kB" avg_uncomp="56 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:26:14.146487307Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:26:14.444278194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="598 B" avg_uncomp="598 B" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:26:14.447190345Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:26:14.744261213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:26:14.747081133Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:26:15.043961457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:26:15.046706013Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:26:15.319563012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:26:15.326505197Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:26:15.344005329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:26:15.346517006Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:26:17.143657779Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:26:17.144716027Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:26:17.164038381Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:26:17.164088445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:26:17.164097454Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:26:17.164097989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:26:17.165190852Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-05T12:26:19.15223021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:26:20.501408391Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T12:26:20.501439767Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=warn ts=2026-05-05T12:26:22.16431257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:26:23.832593667Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T12:26:25.136458145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:26:25.144732592Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:26:25.42041659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:26:25.423164691Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-05T12:26:26.229705548Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:49506"
level=info ts=2026-05-05T12:26:26.507815423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:26:26.510823535Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:26:26.854106581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:26:26.857207655Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:26:27.164920225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:26:27.236342296Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-05T12:26:27.236374067Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=2
level=info ts=2026-05-05T12:26:28.283571904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:26:28.286457134Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:26:28.846100006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:26:28.850032949Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:26:30.172664252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:26:30.175784742Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:26:30.498915291Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-05T12:26:30.615300907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:26:30.618536026Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T12:26:32.165242017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:26:32.265495777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:26:32.272919236Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:26:33.254559365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:26:33.258685833Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:26:33.831403646Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-05T12:26:37.16488743Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T12:26:37.164894697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:26:40.498677696Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-05T12:26:42.165176185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:26:42.211465707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:26:42.211516285Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-05T12:26:42.214826761Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:26:42.963459371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:26:42.971416445Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:26:43.831483029Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T12:26:44.187760639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:26:44.191761853Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:26:47.164806598Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:26:47.164842924Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:26:47.164853195Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:26:47.164857465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:26:47.164919786Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
ts=2026-05-05T12:26:49.154360984Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:26:50.498947848Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:26:50.498971765Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-05T12:26:52.164870112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:26:53.832006819Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-05-05T12:26:54.103554908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:26:54.106955851Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:26:56.037165973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:26:56.039882102Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:26:56.673841352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:26:56.677573099Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:26:57.051917646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:26:57.054689149Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:26:57.164141963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:26:57.165802464Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:26:57.368636354Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:26:58.861662482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:26:58.864741229Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:26:59.185326135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:26:59.188229858Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:27:00.498638079Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-05T12:27:00.655767803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:27:00.658726219Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T12:27:02.164608385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:27:03.831825114Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-05T12:27:04.389284312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:27:04.392040733Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:27:05.925009302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:27:05.92947038Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:27:06.170489715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:27:06.174199646Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:27:07.164559353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:27:07.165182607Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:27:07.165202257Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-05T12:27:08.20736901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:27:08.214772941Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:27:08.881675643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:27:08.884315338Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:27:09.775273106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:27:09.783471134Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:27:09.878699446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:27:09.886448693Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:27:10.182092579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:27:10.51053152Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-05T12:27:10.510555809Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=warn ts=2026-05-05T12:27:12.165028321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:27:13.831423929Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-05T12:27:17.143895511Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:27:17.144072225Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:27:17.164410374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:27:17.16444211Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:27:17.164455182Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:27:17.164488805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:27:17.165564768Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
ts=2026-05-05T12:27:18.267794733Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35414"
ts=2026-05-05T12:27:19.155834412Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:27:20.498912134Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:27:22.164807801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:27:23.831638153Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=info ts=2026-05-05T12:27:24.124283271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:27:24.133969551Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:27:24.133996066Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:27:24.134042265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:27:24.142359047Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:27:26.011103531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:27:26.015843358Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:27:26.015870563Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:27:26.015904154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:27:26.019239778Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T12:27:27.164566029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:27:27.1646949Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-05T12:27:29.265953826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34972"
level=debug ts=2026-05-05T12:27:30.4982964Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-05T12:27:32.164488769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:27:33.457209827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:27:33.460705381Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:27:33.831725899Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-05T12:27:34.995398251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:27:35.001897462Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:27:37.164973577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:27:37.165137704Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-05T12:27:38.181402453Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:27:40.308925519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:27:40.316538143Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:27:40.498403749Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-05T12:27:42.164465661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:27:42.795291878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:27:42.798632798Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:27:43.831960181Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T12:27:43.831982978Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-05T12:27:44.2831229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:27:44.286709384Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:27:45.682592824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:27:45.685616654Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:27:47.105421462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:27:47.108582835Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:27:47.164413915Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:27:47.164445102Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:27:47.164453751Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:27:47.164458949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:27:47.165844428Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
level=info ts=2026-05-05T12:27:48.902118386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:27:48.905047614Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-05T12:27:49.157301559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:27:49.305198537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:27:49.308761924Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:27:50.089493936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:27:50.097206716Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:27:50.265121246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:27:50.26893263Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:27:50.499647757Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-05T12:27:50.499678211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=info ts=2026-05-05T12:27:51.051067818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:27:51.053675171Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T12:27:51.720795553Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:27:52.164431081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:27:53.085507837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:27:53.088216982Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:27:53.832003167Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-05T12:27:56.943827443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:27:56.95254538Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:27:57.164907868Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=warn ts=2026-05-05T12:27:57.164937068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:27:58.85489656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:27:58.857818744Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:28:00.498428453Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-05T12:28:02.165175572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:28:03.832152919Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-05T12:28:05.476193368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:28:05.481403212Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:28:05.681904678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:28:05.68866689Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:28:05.688691363Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:28:05.688723128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:28:05.696237539Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:28:07.070169778Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:28:07.164607262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:28:07.164709967Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-05-05T12:28:08.20892101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:28:08.213102985Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:28:10.315130303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:28:10.323048863Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:28:10.498794946Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-05T12:28:12.164802505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:28:13.831371155Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=debug ts=2026-05-05T12:28:17.14366537Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:28:17.144729569Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:28:17.164076473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:28:17.164106011Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:28:17.16409845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:28:17.164116305Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:28:17.165264755Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
ts=2026-05-05T12:28:18.271209901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60162"
ts=2026-05-05T12:28:19.158834557Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:28:20.498463902Z caller=checkpoint.go:372 msg="writing series" size="415 kB"
level=info ts=2026-05-05T12:28:21.34884192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:28:21.358979107Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:28:22.164288053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:28:23.832088901Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T12:28:24.879369862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:28:24.883609295Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:28:27.164157073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:28:27.168185066Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:28:27.168207389Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=11
level=info ts=2026-05-05T12:28:30.054671885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:28:30.064688203Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:28:30.49818177Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-05T12:28:31.844936535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:28:31.847740159Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:28:31.922419553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:28:31.92527861Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:28:32.164563125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:28:33.832136142Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-05T12:28:35.956242505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:28:35.962104194Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:28:35.962128471Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:28:35.962163286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:28:35.965375379Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:28:37.164834467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:28:37.164934571Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-05T12:28:39.201114975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:28:40.107611989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:28:40.107668163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-05T12:28:40.114946798Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:28:40.498652917Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:28:40.801656844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:28:40.804341306Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:28:41.619404362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:28:41.622418093Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T12:28:42.16479937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:28:42.660867234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="548 B" avg_comp="548 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:28:42.663480514Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:28:43.83232941Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=warn ts=2026-05-05T12:28:47.164201671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:28:47.164203255Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:28:47.164242519Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:28:47.164255461Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:28:47.165372772Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-05T12:28:48.054937968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:28:48.063024348Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
ts=2026-05-05T12:28:48.272405405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:38714"
level=info ts=2026-05-05T12:28:48.6525996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:28:48.656209674Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:28:48.801634681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:28:48.804945472Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:28:49.068646501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:28:49.071939174Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:28:49.071963017Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:28:49.071992355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:28:49.074723876Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T12:28:49.161407152Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:28:49.490602708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:28:49.493558884Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:28:50.023038331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:28:50.031801636Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:28:50.031836578Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:28:50.031875243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:28:50.03833446Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:28:50.370421719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:28:50.37671763Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:28:50.498193328Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:28:50.498220642Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-05T12:28:52.16431981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:28:53.831694499Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:28:54.212923945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:28:54.216470161Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:28:56.836414262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:28:56.842718805Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:28:57.164767335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:28:57.179924616Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:28:57.179966192Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=info ts=2026-05-05T12:28:58.04089054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:28:58.048126455Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:29:00.49910041Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-05T12:29:00.986794907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:29:00.992628036Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:29:02.164346338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:29:03.38310962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:29:03.390437234Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:29:03.83255848Z caller=checkpoint.go:372 msg="writing series" size="921 kB"
level=debug ts=2026-05-05T12:29:03.832584389Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-05T12:29:04.384370383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:29:04.387637958Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:29:07.164427853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:29:07.16925388Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-05T12:29:07.16928769Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=1
level=debug ts=2026-05-05T12:29:10.498519633Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=warn ts=2026-05-05T12:29:12.164302604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:29:13.136182991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:29:13.140638152Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:29:13.709390324Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:29:13.831842165Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:29:17.14412695Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:29:17.144164599Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:29:17.164551287Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:29:17.164581755Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:29:17.16458183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:29:17.164592726Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:29:17.16754035Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-05T12:29:17.167557372Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=info ts=2026-05-05T12:29:17.7104341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:29:17.714544894Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:29:18.27428874Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:42438"
level=info ts=2026-05-05T12:29:18.964030861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:29:18.967844107Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-05T12:29:19.16431568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:29:20.498247195Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=warn ts=2026-05-05T12:29:22.164621935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:29:23.17818674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:29:23.185128368Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:29:23.832205064Z caller=checkpoint.go:372 msg="writing series" size="399 kB"
level=info ts=2026-05-05T12:29:24.381628701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:29:24.38444329Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:29:24.936019771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:29:24.93875953Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:29:27.039363771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:29:27.041786434Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T12:29:27.164758815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:29:27.164783049Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-05T12:29:28.274980479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:29:28.277818555Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:29:28.405177927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:29:28.408240846Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:29:28.661036366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:29:28.669400624Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:29:28.901577454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:29:29.277299664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:45618"
level=debug ts=2026-05-05T12:29:30.499420372Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:29:30.499443464Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=4
level=warn ts=2026-05-05T12:29:32.164436631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:29:33.184579607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:29:33.191850308Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:29:33.831548914Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=info ts=2026-05-05T12:29:36.85681198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:29:36.859639551Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:29:37.059325038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:29:37.062001862Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T12:29:37.164476313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:29:37.164677826Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-05-05T12:29:39.687409954Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:29:40.499029499Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-05T12:29:42.164615157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:29:43.831659751Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-05T12:29:43.966107822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:29:43.969808704Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:29:45.742761837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:29:45.745566444Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:29:47.165007697Z caller=checkpoint.go:384 msg="flushing series" totalSize="720 kB" series=4
level=info ts=2026-05-05T12:29:47.16502674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:29:47.165063251Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:29:47.165072119Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:29:47.165095019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:29:47.172800684Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-05T12:29:47.174151258Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009645243s
ts=2026-05-05T12:29:49.167453807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:29:51.098916305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:29:51.114794226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:29:52.165077378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:29:53.327123743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:29:53.334775912Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:29:54.964677851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:29:54.969042574Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:29:57.164792399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:30:01.40735606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:30:01.411141446Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:30:02.164997256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:30:03.03912641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:30:03.046020837Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:30:04.238795375Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:30:05.273091618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:30:05.276222661Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:30:06.999889053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:30:07.006373823Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T12:30:07.164666123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:30:08.489992642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:30:08.493107809Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:30:12.164350272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:30:15.853838143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:30:15.856950787Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:30:17.143787437Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:30:17.14486187Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:30:17.145139659Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-05T12:30:17.14518104Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975537 n=1
level=debug ts=2026-05-05T12:30:17.145221648Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975537 n=1
level=info ts=2026-05-05T12:30:17.164158498Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T12:30:17.164169263Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:30:17.164190988Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:30:17.164201258Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:30:17.164221333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:30:17.164299236Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-05T12:30:17.164308779Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:30:17.164330099Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.687µs success=true
level=debug ts=2026-05-05T12:30:17.16454819Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-05-05T12:30:18.27963078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:47184"
ts=2026-05-05T12:30:19.169517484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:30:20.253971403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:30:20.256973571Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:30:20.499430139Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=info ts=2026-05-05T12:30:21.996111021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:30:22.000853129Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:30:22.164985615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:30:22.777545354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:30:22.780333843Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:30:23.831429571Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-05T12:30:25.671870291Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:30:25.673426663Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T12:30:27.050523563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:30:27.05902804Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:30:27.164596313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:30:27.164619492Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-05T12:30:27.164637109Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-05T12:30:27.499880562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:30:27.502611669Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:30:28.556107784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:30:28.558734605Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:30:28.783335836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:30:28.791605904Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:30:29.540336144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:30:29.543925984Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:30:30.497946141Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-05T12:30:30.521016895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:30:30.523516054Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T12:30:32.164928169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:30:33.831883782Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-05T12:30:36.615390399Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:30:37.16493205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:30:37.164968646Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=debug ts=2026-05-05T12:30:40.498867641Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-05T12:30:42.16435654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:30:43.394322371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:30:43.405968782Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:30:43.832613869Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=info ts=2026-05-05T12:30:47.164791415Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:30:47.164827611Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:30:47.164839794Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:30:47.164905051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:30:47.165015451Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:30:47.165031454Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-05T12:30:47.191042259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:30:47.19969874Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:30:49.017947993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:30:49.021545572Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-05T12:30:49.171947057Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:30:49.598152313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:30:49.602634992Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:30:50.498235957Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=info ts=2026-05-05T12:30:51.442702798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:30:51.446404104Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:30:52.164842679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:30:53.365368991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:30:53.371001953Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:30:53.371025986Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:30:53.371071331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:30:53.374467256Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:30:53.831674569Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:30:56.499904249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:30:56.503002163Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:30:57.164602988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:30:57.167127496Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-05T12:30:57.167148467Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=info ts=2026-05-05T12:30:57.245960861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:30:57.253078921Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:31:00.499122341Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-05T12:31:02.164534579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:31:02.772199741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:31:02.775467653Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:31:03.019100734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:31:03.022485162Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:31:03.832447548Z caller=checkpoint.go:372 msg="writing series" size="456 kB"
level=info ts=2026-05-05T12:31:05.213205391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:31:05.216879676Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:31:05.713373915Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:31:07.164495223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:31:07.164589512Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-05T12:31:10.116999777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:31:10.122861084Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:31:10.498221081Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T12:31:12.143468051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:31:12.146892466Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:31:12.156947303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:31:12.163805262Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:31:12.164794588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:31:12.432040511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:31:12.434699701Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T12:31:13.831371192Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-05T12:31:14.499540578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:31:15.406432755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:31:15.409895565Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:31:16.07077384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:31:16.073306543Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:31:17.143356327Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:31:17.144394853Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:31:17.164785168Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:31:17.164796344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:31:17.16481314Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:31:17.16482364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:31:17.164880271Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-05-05T12:31:17.164892374Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-05T12:31:18.956764119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:31:18.959428988Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-05T12:31:19.173365339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:31:20.501508188Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T12:31:20.501529046Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-05T12:31:20.838017776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:31:20.845181404Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:31:22.165192816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:31:23.831621432Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T12:31:23.837747839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:31:23.844486976Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:31:27.165000024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:31:27.231651362Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-05T12:31:27.231683323Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=2
level=debug ts=2026-05-05T12:31:30.498259191Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-05T12:31:32.164299745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:31:33.831510118Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-05T12:31:37.164787724Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T12:31:37.164821744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:31:37.256112983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="90 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:31:37.265650987Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:31:37.438673935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:31:37.442421896Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:31:39.560187437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:31:39.563246365Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:31:40.222263177Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:31:40.498146696Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-05T12:31:40.866423951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:31:40.871151734Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:31:41.089715818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:31:41.09436475Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:31:41.094386635Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:31:41.09440827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:31:41.0974077Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:31:42.164621638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:31:43.832237284Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T12:31:44.691114823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:31:44.694546475Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:31:47.164551003Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:31:47.164586309Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:31:47.164595638Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:31:47.164612632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:31:47.164700713Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=info ts=2026-05-05T12:31:47.856094654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:31:47.859818143Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:31:49.174998715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:31:50.49975547Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:31:50.499796211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-05T12:31:51.500459268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:31:51.504566293Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:31:52.164785977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:31:53.18776271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:31:53.195311884Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:31:53.832111595Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-05-05T12:31:53.867988265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:31:53.871390928Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:31:55.025620499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:31:55.033082356Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:31:57.165003806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:31:57.165680228Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=info ts=2026-05-05T12:31:58.788094598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:31:58.79116482Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:32:00.312172302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:32:00.3151801Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:32:00.498480548Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-05T12:32:02.164334147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:32:03.322956109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:32:03.326724238Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:32:03.832196073Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-05T12:32:07.164477432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:32:07.165272985Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:32:07.165291423Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=debug ts=2026-05-05T12:32:09.382157516Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:32:09.706946404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:32:10.514949781Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:32:10.514976743Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=1
level=info ts=2026-05-05T12:32:10.763417017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:32:10.766001649Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:32:12.084290937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:32:12.098316188Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T12:32:12.165043131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:32:13.222282214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:32:13.226258089Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:32:13.832255263Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-05T12:32:15.461334947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:32:15.46870235Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:32:16.40403928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:32:16.40681444Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:32:17.144014642Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:32:17.144046182Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:32:17.16437444Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:32:17.164401921Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:32:17.164411651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:32:17.164455704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:32:17.164542538Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
ts=2026-05-05T12:32:19.176750787Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:32:19.555886393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:32:19.563252169Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:32:20.49894059Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:32:22.164588568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:32:23.831597172Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-05-05T12:32:25.436847445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:32:25.444528095Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:32:27.165009859Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-05T12:32:27.165025839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:32:30.499046589Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:32:31.281481506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:32:31.284289072Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:32:32.164775655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:32:33.831652074Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-05T12:32:35.759252531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:32:35.762383181Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:32:35.762404747Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:32:35.762425585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:32:35.764416074Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:32:36.551560323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:32:36.554810638Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:32:36.554831135Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:32:36.554852795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:32:36.55679108Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T12:32:37.16488841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:32:37.164986933Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=debug ts=2026-05-05T12:32:37.247336355Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:32:38.640735399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:32:38.643885131Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:32:40.498298183Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-05T12:32:40.661902065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:32:40.665352925Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:32:42.164887188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:32:42.578829591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:32:42.582392306Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:32:42.582410861Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:32:42.582432423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:32:42.584504798Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:32:43.62435036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:32:43.627298785Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:32:43.832365167Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T12:32:43.832387804Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-05-05T12:32:47.164785066Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:32:47.164819349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:32:47.164829865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:32:47.16483531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:32:47.165156903Z caller=checkpoint.go:372 msg="writing series" size="586 kB"
ts=2026-05-05T12:32:48.291233478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:58702"
level=info ts=2026-05-05T12:32:48.406007216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:32:48.410311988Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:32:48.473496524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:32:48.480502863Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-05T12:32:49.178786501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:32:49.903260052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:32:49.905938653Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:32:50.499422464Z caller=checkpoint.go:372 msg="writing series" size="900 kB"
level=debug ts=2026-05-05T12:32:50.499444255Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-05-05T12:32:50.531880064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:32:50.539466959Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:32:52.165110391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:32:53.221714962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:32:53.224961815Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:32:53.831611216Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-05T12:32:56.178657806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:32:56.181942721Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:32:56.18196691Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:32:56.181993798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:32:56.184386111Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T12:32:57.164666798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:32:57.164737544Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-05T12:32:58.899017425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:32:58.901801442Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:33:00.498613963Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-05T12:33:02.164851349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:33:02.295243533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:33:02.299350882Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:33:03.523155799Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:33:03.832147846Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-05T12:33:05.724044968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:33:05.726646991Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:33:05.726667109Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:33:05.72668856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:33:05.72877954Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T12:33:07.16453805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:33:07.164646349Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=debug ts=2026-05-05T12:33:07.221879922Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:33:07.683742428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:33:07.691769326Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:33:09.242526434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:33:09.24516214Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:33:09.591177609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:33:09.593822233Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-05T12:33:09.593843378Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:33:09.593865756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:33:09.596031994Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:33:10.21833551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:33:10.225297124Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:33:10.225318027Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:33:10.225349114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:33:10.232531697Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:33:10.498924512Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=warn ts=2026-05-05T12:33:12.164577325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:33:13.620322325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:33:13.63152582Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:33:13.83201833Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-05T12:33:13.964454944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:33:13.967080043Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:33:13.967101381Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:33:13.96712293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:33:13.969139582Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:33:16.196731197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:33:16.199220078Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:33:16.790110488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:33:16.79272811Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:33:16.792748263Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:33:16.792770919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:33:16.795004109Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:33:17.143794274Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:33:17.144861089Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:33:17.164172655Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:33:17.164194873Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:33:17.164186913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:33:17.164203972Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:33:17.165366754Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
ts=2026-05-05T12:33:19.180374603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:33:20.498994418Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-05T12:33:20.499017088Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-05T12:33:21.901253653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:33:21.903749717Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:33:21.909813165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:33:21.912128459Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T12:33:22.164685747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:33:22.67352933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:33:22.676108154Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:33:22.899713871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="548 B" avg_comp="548 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:33:22.902342231Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:33:23.007717706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:33:23.011280246Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:33:23.83187497Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-05T12:33:26.263388073Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:45628"
level=info ts=2026-05-05T12:33:26.845330684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:33:26.848092943Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T12:33:27.164357249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:33:27.168035146Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:33:27.168069192Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-05-05T12:33:28.05543497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="429 kB" avg_comp="429 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:33:28.055500954Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-05T12:33:28.05870086Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:33:28.839108256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:33:28.8433765Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:33:29.339047531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:33:29.3475124Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:33:30.498582422Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-05T12:33:32.165140402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:33:33.753484337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:33:33.756454608Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:33:33.831584975Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-05T12:33:34.398464308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:33:34.404962347Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T12:33:37.164969388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:33:37.165160944Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-05T12:33:37.632798635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:33:37.636355647Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:33:40.498394852Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T12:33:42.164942594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:33:43.184194336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:33:43.187943734Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:33:43.639244283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:33:43.644113094Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:33:43.832145519Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-05-05T12:33:44.715381414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:33:44.722617106Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:33:45.761257703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:33:45.766419273Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:33:45.766442476Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:33:45.766466451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:33:45.769348582Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:33:47.164345118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:33:47.164388141Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:33:47.164399909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:33:47.164394656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:33:47.164619393Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-05T12:33:48.295960301Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59716"
ts=2026-05-05T12:33:49.182774678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:33:50.499009358Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:33:50.499032868Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-05T12:33:51.642507282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:33:51.646184719Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:33:52.164346552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:33:52.204886258Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:33:52.30711344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:33:52.314646612Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:33:52.553847084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:33:52.556481155Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:33:53.561880445Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:33:53.831663296Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-05-05T12:33:56.265305957Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:35000"
level=info ts=2026-05-05T12:33:56.362078423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:33:56.364849566Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:33:56.740673434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:33:56.743198974Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:33:57.120174086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:33:57.122963108Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T12:33:57.164975466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:33:57.175462735Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:33:57.175488452Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=debug ts=2026-05-05T12:34:00.499028002Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=info ts=2026-05-05T12:34:01.532201496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:34:01.54002381Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:34:02.165136575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:34:03.691612963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:34:03.695107415Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:34:03.831904111Z caller=checkpoint.go:372 msg="writing series" size="937 kB"
level=debug ts=2026-05-05T12:34:03.83193291Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-05T12:34:07.164544196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:34:07.167945762Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-05-05T12:34:07.167966135Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=1
level=info ts=2026-05-05T12:34:10.209357214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:34:10.216193005Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:34:10.216215976Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:34:10.216256343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:34:10.223166703Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:34:10.498916277Z caller=checkpoint.go:372 msg="writing series" size="935 B"
level=info ts=2026-05-05T12:34:11.688901165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:34:11.691771166Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:34:12.067136202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:34:12.070373249Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:34:12.164432406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:34:13.784762779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:34:13.790757062Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:34:13.83190841Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:34:15.038453908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:34:15.041666502Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:34:16.230287219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:34:16.233281796Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:34:17.143837124Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:34:17.144892139Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:34:17.16435093Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:34:17.164378697Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:34:17.164387851Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:34:17.16439658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:34:17.168515179Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:34:17.168538399Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=3
ts=2026-05-05T12:34:19.185754712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:34:20.498451223Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=info ts=2026-05-05T12:34:21.569065934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:34:21.574577353Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:34:22.02339992Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:34:22.165263166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:34:23.83235688Z caller=checkpoint.go:372 msg="writing series" size="511 kB"
level=info ts=2026-05-05T12:34:23.890144133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:34:23.899830843Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:34:26.841591557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:34:26.845355725Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:34:27.164912329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:34:27.164939629Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-05T12:34:27.99696686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:34:28.004095128Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:34:30.394715794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:34:30.397622852Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:34:30.499892384Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:34:30.499915442Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=warn ts=2026-05-05T12:34:32.165108488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:34:32.565818175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:34:32.573746461Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:34:33.831260594Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-05-05T12:34:34.3639524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:34:34.371538047Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:34:37.164562076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:34:37.164683048Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-05T12:34:37.388718662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:34:37.392556825Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:34:38.409342728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:34:38.413017114Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:34:38.682829991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:34:38.686386079Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:34:40.498380082Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=info ts=2026-05-05T12:34:42.006129262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:34:42.008554807Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:34:42.011687881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:34:42.013417236Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T12:34:42.164123784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:34:43.832225412Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=debug ts=2026-05-05T12:34:47.164753693Z caller=checkpoint.go:384 msg="flushing series" totalSize="780 kB" series=4
level=warn ts=2026-05-05T12:34:47.164782679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:34:47.164815504Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:34:47.164826385Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:34:47.164833909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T12:34:47.172783937Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-05T12:34:47.174220669Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009746653s
level=info ts=2026-05-05T12:34:47.704530733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:34:47.707627289Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:34:48.964786944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:34:48.967898738Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-05T12:34:49.187967333Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:34:51.142798131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:34:51.151151995Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:34:51.151176972Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:34:51.151204584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:34:51.157624768Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:34:52.164732781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:34:55.014960481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:34:55.023322282Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-05T12:34:56.270132056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34514"
level=warn ts=2026-05-05T12:34:57.165119846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:34:57.766924091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:34:57.766984691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="198 B" avg_comp="198 B" total_uncomp="615 B" avg_uncomp="615 B" idle=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:34:57.769926139Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:34:58.904978552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:34:58.907830304Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:35:00.117595435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:35:00.120572934Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:35:00.872406346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:35:00.875550774Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:35:02.164705385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:35:07.164860393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:35:10.725976129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:35:10.730210207Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:35:10.73024362Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:35:10.73027801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:35:10.733493035Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:35:11.007348236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:35:11.014977301Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:35:11.3348842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:35:11.337470639Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:35:11.428590415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:35:11.431340285Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:35:11.479402492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:35:11.482511815Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=warn ts=2026-05-05T12:35:12.164393532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:35:14.9284774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:35:14.931972095Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:35:17.144003574Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:35:17.144008853Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:35:17.164389703Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:35:17.16442186Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:35:17.164430977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T12:35:17.164434366Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-05T12:35:17.164492144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:35:17.164620826Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-05T12:35:17.16463022Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:35:17.164649362Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.207µs success=true
level=debug ts=2026-05-05T12:35:17.164899148Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-05T12:35:17.64711884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:35:17.655386928Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:35:17.655416833Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:35:17.655444449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:35:17.662230189Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-05T12:35:18.302877442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:50558"
ts=2026-05-05T12:35:19.190194789Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:35:20.499520783Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=info ts=2026-05-05T12:35:20.860702654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:35:20.864360833Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:35:20.864387685Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:35:20.864419422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:35:20.867651103Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:35:21.423636338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:35:21.426933066Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:35:22.164424219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:35:23.625826051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:35:23.633402533Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:35:23.75981353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:35:23.763254833Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:35:23.831670042Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-05T12:35:25.663939923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:35:25.666581154Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T12:35:27.164710954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:35:27.165836802Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=debug ts=2026-05-05T12:35:27.165853468Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-05T12:35:28.82980519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:35:28.833952918Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:35:29.756254297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:35:29.765221907Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:35:29.765246898Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:35:29.765276278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:35:29.772090491Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:35:30.498769068Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=info ts=2026-05-05T12:35:31.964020298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:35:31.966598236Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:35:32.095869005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:35:32.098573651Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:35:32.164379872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:35:32.675436942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:35:32.679161635Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:35:32.724989299Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:35:33.077358897Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:35:33.831721785Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-05T12:35:34.626996172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:35:34.629764632Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:35:35.841497474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:35:35.848151455Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T12:35:37.164583065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:35:37.165822718Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-05-05T12:35:40.499304173Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-05T12:35:42.165121632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:35:43.83338213Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=info ts=2026-05-05T12:35:44.472516538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:35:44.475575844Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:35:44.831716884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:35:44.840940685Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:35:45.194968626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:35:45.198117306Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:35:45.981973007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:35:45.990727482Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:35:46.233637852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:35:46.240477945Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:35:47.160678806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=info ts=2026-05-05T12:35:47.164091761Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:35:47.164124673Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:35:47.164134692Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:35:47.164135754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-05T12:35:47.164355575Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:35:47.165576329Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-05T12:35:47.165587198Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-05T12:35:49.192412999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:35:50.498958446Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-05-05T12:35:52.165036597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:35:53.832139824Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-05T12:35:56.275044659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:46226"
level=warn ts=2026-05-05T12:35:57.16490086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:35:57.167006504Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:35:57.167035309Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=info ts=2026-05-05T12:35:57.952551797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:35:57.961421108Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:35:58.908272174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:35:58.911668056Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-05T12:35:59.313317883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38894"
level=debug ts=2026-05-05T12:36:00.498768618Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=debug ts=2026-05-05T12:36:00.935581873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:36:01.9953236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:36:01.998705133Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:36:02.164304144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:36:02.756140734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:36:02.759081388Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:36:03.345048498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:36:03.34983654Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:36:03.832608497Z caller=checkpoint.go:372 msg="writing series" size="484 kB"
level=warn ts=2026-05-05T12:36:07.164474763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:36:07.165700626Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-05T12:36:09.46724739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:36:09.47183225Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:36:09.528879955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:36:09.535874494Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:36:09.723244632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:36:09.726039685Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:36:10.498264963Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T12:36:10.916705171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:36:10.920408308Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:36:12.164516424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:36:13.665731765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:36:13.672507258Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:36:13.832128129Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-05T12:36:13.8321522Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T12:36:15.400292297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:36:15.40365238Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=debug ts=2026-05-05T12:36:17.143681891Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:36:17.144756116Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:36:17.164167344Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:36:17.164196356Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:36:17.164204996Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:36:17.164256278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:36:17.165343575Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-05T12:36:17.990442481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:36:17.997153567Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:36:18.587916135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:36:18.591798254Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-05T12:36:19.193931414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:36:19.962924429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:36:19.967082888Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:36:20.500218973Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T12:36:20.500243539Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=warn ts=2026-05-05T12:36:22.164516882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:36:23.832144012Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-05T12:36:27.164572154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:36:27.231872702Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-05T12:36:27.231907442Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=2
level=info ts=2026-05-05T12:36:29.421214945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:36:29.428510579Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:36:30.498622605Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-05T12:36:31.37760696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:36:31.381233386Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:36:31.981146329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:36:32.164310932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:36:33.741080287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:36:33.744207092Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T12:36:33.832488192Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-05T12:36:35.653908084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:36:35.66267266Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:36:36.017812447Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:36:36.017851816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:36:36.021498276Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T12:36:37.164571726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:36:37.165683618Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-05T12:36:37.362949561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:36:37.366142733Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:36:38.090527558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:36:38.099333786Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:36:40.499261461Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-05T12:36:42.164810403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:36:43.698641561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:36:43.701679348Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:36:43.832186939Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T12:36:47.164630679Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:36:47.164666962Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:36:47.164654881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:36:47.164675246Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:36:47.165863557Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
ts=2026-05-05T12:36:48.310363855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:32974"
level=info ts=2026-05-05T12:36:48.697536393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:36:48.701753165Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:36:49.196643865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:36:50.502277945Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:36:50.502329084Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-05T12:36:50.738749069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:36:50.744707053Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:36:52.164974312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:36:52.636874263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:36:52.639831435Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:36:53.832615351Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=info ts=2026-05-05T12:36:54.700905219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:36:54.704232742Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:36:54.793580745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:36:54.797513493Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T12:36:57.164486355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:36:57.165952832Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=info ts=2026-05-05T12:36:58.319116703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:36:58.321893594Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:36:58.547779179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:36:58.550575878Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:37:00.28892305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:37:00.292640569Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=debug ts=2026-05-05T12:37:00.498670108Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-05T12:37:00.839474943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:37:00.843508602Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:37:01.749658052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:37:01.757252731Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:37:02.164521502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:37:02.18266928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:37:02.186278494Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:37:02.931706901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:37:02.935282978Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:37:03.8320006Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:37:06.276496348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:37:06.283354697Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T12:37:07.164620788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:37:07.166412904Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:37:07.166438139Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=debug ts=2026-05-05T12:37:10.508795351Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-05T12:37:10.508820965Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=debug ts=2026-05-05T12:37:11.248249999Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:37:11.442786131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:37:11.450771148Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:37:12.165141553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:37:13.271149372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:37:13.275457427Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:37:13.832241436Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-05T12:37:17.143965716Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:37:17.143971679Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:37:17.164414248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:37:17.16441843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:37:17.164441742Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:37:17.164451206Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:37:17.165665684Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
ts=2026-05-05T12:37:19.199299697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:37:20.49897406Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-05T12:37:21.111612122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:37:21.116334825Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-05T12:37:22.165119693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:37:23.117046711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:37:23.120009365Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:37:23.544829508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:37:23.553265547Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:37:23.743869331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:37:23.746621856Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:37:23.83333071Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-05T12:37:24.545081715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:37:24.550387597Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T12:37:27.164214929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:37:27.165443742Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-05T12:37:28.634691941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:37:28.63783225Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:37:30.33282009Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:37:30.499382006Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-05T12:37:32.164986656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:37:33.831917585Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-05T12:37:36.560171342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:37:36.566803278Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:37:37.164707892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:37:37.165901635Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-05-05T12:37:38.784708151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:37:38.788382444Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:37:40.499262977Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-05T12:37:42.164342316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:37:42.206517786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:37:42.210290409Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:37:43.649923682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:37:43.652700006Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:37:43.813310414Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:37:43.832611939Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T12:37:43.832633132Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-05T12:37:44.224645232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:37:44.231694239Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:37:44.532509251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:37:44.535484483Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:37:47.164793065Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:37:47.164826575Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:37:47.164835704Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:37:47.16485798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:37:47.16528626Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=info ts=2026-05-05T12:37:47.803660888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:37:47.811208121Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-05T12:37:49.200875615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:37:50.115198947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:37:50.118774622Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:37:50.499646665Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-05T12:37:50.499670802Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=warn ts=2026-05-05T12:37:52.164150242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:37:52.322547038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:37:52.325249395Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:37:52.46693062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:37:52.469622606Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:37:53.83181533Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-05T12:37:54.224954138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:37:54.229003914Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:37:54.229034457Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:37:54.229086003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:37:54.232297178Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:37:56.334239059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:37:56.337922985Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:37:57.164514069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:37:57.165666905Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-05T12:37:58.718527177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:37:58.722488608Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:37:58.722513559Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:37:58.72254317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:37:58.725751665Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:38:00.498783459Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-05T12:38:02.165143199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:38:02.575573714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:38:02.5793386Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:38:03.80575094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:38:03.808546297Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:38:03.831643999Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-05T12:38:05.700169752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:38:05.704777013Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:38:07.165022284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:38:07.165126179Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
level=info ts=2026-05-05T12:38:08.914129377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:38:08.917618371Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:38:09.994012847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:38:10.001778314Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:38:10.499108073Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-05-05T12:38:12.164534109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:38:13.549927067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:38:13.557673878Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:38:13.832418958Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-05T12:38:15.483518694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:38:15.487269495Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:38:17.143558139Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:38:17.144607361Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:38:17.164529293Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:38:17.164561792Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:38:17.164570587Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:38:17.164644341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:38:17.16577828Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=info ts=2026-05-05T12:38:18.320583281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:38:18.323675146Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:38:18.512282984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:38:18.515076187Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-05T12:38:19.202677619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:38:19.781308788Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:38:20.498780653Z caller=checkpoint.go:372 msg="writing series" size="389 kB"
level=debug ts=2026-05-05T12:38:20.498805506Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-05T12:38:20.81965604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:38:20.823483596Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T12:38:22.164870674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:38:23.832479389Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T12:38:23.925633963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:38:23.929375032Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:38:25.543247068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:38:25.547125173Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:38:26.286180293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:39886"
level=warn ts=2026-05-05T12:38:27.164670164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:38:27.16845102Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:38:27.168470785Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=debug ts=2026-05-05T12:38:30.499077576Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-05T12:38:30.613442699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:38:30.619711856Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:38:31.556449554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:38:31.559108153Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=warn ts=2026-05-05T12:38:32.16457952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:38:32.498683331Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:38:32.814675585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:38:32.817946005Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:38:33.496967622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:38:33.503923073Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:38:33.831871978Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-05T12:38:36.89991509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:38:36.903077169Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T12:38:37.164895349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:38:37.16495887Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-05T12:38:37.906602074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:38:37.910042674Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:38:38.437866991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:38:38.441140881Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:38:40.498437973Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:38:41.543820162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:38:41.547265267Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:38:41.718192454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:38:41.725492582Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:38:42.164872477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:38:43.220768758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:38:43.22383581Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:38:43.832234411Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-05-05T12:38:46.235037788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:38:46.238543709Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:38:46.238568235Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:38:46.238593359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:38:46.242114123Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:38:46.286273056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:38:46.289826341Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:38:47.164201404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:38:47.164204436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:38:47.164246805Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:38:47.164254436Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:38:47.165437909Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-05T12:38:49.203565444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:38:50.499127963Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:38:50.499153039Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-05T12:38:51.460380043Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:38:51.460446709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="12 kB" avg_uncomp="12 kB" idle=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:38:51.463158303Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=warn ts=2026-05-05T12:38:52.164988735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:38:53.832111772Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T12:38:57.164246874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:38:57.1766005Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:38:57.176627247Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
ts=2026-05-05T12:38:59.329840812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:56228"
level=debug ts=2026-05-05T12:39:00.498542776Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=info ts=2026-05-05T12:39:01.753086131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:39:01.757618751Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:39:01.757643037Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:39:01.757667064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:39:01.759965957Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:39:01.879301422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:39:01.882221059Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:39:02.164765071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:39:02.406528821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:39:02.409282204Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:39:03.061493327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:39:03.071135265Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:39:03.487451543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:39:03.490150411Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:39:03.697781428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:39:03.700502646Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:39:03.78992507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="548 B" avg_comp="548 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:39:03.793212783Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:39:03.83286931Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=debug ts=2026-05-05T12:39:03.832891785Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-05T12:39:04.465295043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:39:04.467925304Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:39:04.467949421Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:39:04.467973692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:39:04.470146849Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:39:07.164405243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:39:07.166742856Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-05T12:39:07.16676027Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=1
level=info ts=2026-05-05T12:39:09.341136924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:39:09.349204933Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:39:10.498377635Z caller=checkpoint.go:372 msg="writing series" size="895 B"
level=warn ts=2026-05-05T12:39:12.165085708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:39:12.637494294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:39:12.640389877Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:39:12.993347312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:39:12.99614646Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T12:39:13.831810974Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:39:15.193887306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="627 B" avg_comp="627 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:39:15.197350069Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:39:17.143891595Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:39:17.144336281Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:39:17.164682746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:39:17.164714546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:39:17.164725733Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:39:17.164735451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:39:17.167208068Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-05T12:39:17.167221764Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=3
level=info ts=2026-05-05T12:39:17.628710185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:39:17.63158659Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:39:17.631605811Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:39:17.631627375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:39:17.634100332Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:39:18.617648898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:39:18.621336125Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:39:19.205585559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:39:20.16054923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:39:20.163751164Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:39:20.498794612Z caller=checkpoint.go:372 msg="writing series" size="318 kB"
level=info ts=2026-05-05T12:39:22.014070904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:39:22.022748997Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:39:22.165098966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:39:22.643478865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:39:22.646931947Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:39:22.874554501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:39:22.881252672Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:39:23.831998198Z caller=checkpoint.go:372 msg="writing series" size="408 kB"
level=debug ts=2026-05-05T12:39:24.425792916Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:39:25.322502395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:39:25.325905362Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-05T12:39:25.325929672Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:39:25.325956712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:39:25.328797419Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:39:26.423101714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:39:26.426573441Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:39:27.077450272Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:39:27.079630715Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T12:39:27.164889505Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-05T12:39:27.164880509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:39:27.479373885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:39:27.482773222Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:39:28.750183258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:39:28.752953759Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-05T12:39:29.332953897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:34620"
level=debug ts=2026-05-05T12:39:30.500620927Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:39:30.500645594Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=4
level=info ts=2026-05-05T12:39:30.871439796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:39:30.879481044Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:39:30.879509877Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:39:30.879543975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:39:30.888325345Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:39:32.164716029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:39:32.490190603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:39:32.494522555Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:39:33.085634372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:39:33.088846767Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:39:33.831882731Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=info ts=2026-05-05T12:39:34.902337771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:39:34.905108161Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:39:35.829859661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:39:35.832434732Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:39:36.063497736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:39:36.066735778Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:39:37.164405858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:39:37.165546362Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-05T12:39:38.052542998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:39:38.055297193Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:39:38.055317922Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:39:38.055340534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:39:38.057405025Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:39:40.498643871Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=warn ts=2026-05-05T12:39:42.164548748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:39:43.832151878Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-05T12:39:44.303610992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:39:44.306574683Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:39:44.311696189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:39:44.314300805Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:39:44.314316197Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:39:44.314339421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:39:44.316835812Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:39:44.83148725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:39:44.838483572Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:39:47.164154813Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:39:47.164187299Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:39:47.164195514Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:39:47.164219189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:39:47.165270816Z caller=checkpoint.go:384 msg="flushing series" totalSize="810 kB" series=4
level=info ts=2026-05-05T12:39:47.172931956Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-05T12:39:47.174300879Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009481393s
level=info ts=2026-05-05T12:39:49.036097695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:39:49.039022054Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:39:49.099271877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:39:49.106312582Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T12:39:49.20818971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=warn ts=2026-05-05T12:39:52.165164283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:39:53.952321442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:39:53.954875192Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:39:55.890672177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:39:55.89386259Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-05T12:39:56.29364193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:37518"
level=info ts=2026-05-05T12:39:56.308837991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:39:56.31142577Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:39:56.31144439Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:39:56.311465089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:39:56.31337994Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:39:57.164836024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:39:59.068645494Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T12:39:59.336282539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:53524"
level=info ts=2026-05-05T12:40:01.595218896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:40:01.599479097Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:40:02.164833055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:40:02.200924293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:40:02.203712879Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:40:02.858549143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:40:02.867256346Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:40:06.049482383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:40:06.054124134Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:40:06.630354555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:40:06.633112096Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:40:07.164374488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:40:08.600465147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:40:08.603130678Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:40:08.70235439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:40:08.705671995Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:40:08.962646722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:40:08.968843462Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:40:09.260680442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:40:09.263311282Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-05T12:40:09.263329034Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:40:09.26335148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:40:09.266785Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:40:10.753274983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:40:10.756391547Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:40:11.73161833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:40:11.738098329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:40:12.164696431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:40:13.02068631Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:40:17.143557435Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:40:17.144624303Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:40:17.164027Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T12:40:17.164027032Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:40:17.164067413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:40:17.164092593Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:40:17.164112118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:40:17.164200268Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-05T12:40:17.164208702Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:40:17.164226122Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.629µs success=true
level=debug ts=2026-05-05T12:40:17.164440643Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-05T12:40:18.993929165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:40:18.997388163Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:40:18.997414554Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:40:18.99744705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:40:19.000124088Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-05T12:40:19.210610176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:40:20.499205449Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=info ts=2026-05-05T12:40:21.866301152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:40:21.870213184Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:40:21.912279307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:40:21.914920516Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=warn ts=2026-05-05T12:40:22.164985067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:40:22.390516707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:40:22.396195278Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:40:23.831118944Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-05T12:40:25.599899824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:40:25.602915799Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:40:26.800181422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:40:26.803187484Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T12:40:27.164635955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:40:27.164750605Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-05T12:40:27.164768689Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-05T12:40:27.37698208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:40:27.379461367Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:40:28.16762677Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:40:28.249784536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:40:28.252365178Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:40:29.988173432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:40:29.990683962Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:40:30.497894002Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-05T12:40:32.165225391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:40:33.123106811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:40:33.127356845Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:40:33.12952101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:40:33.13239037Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:40:33.831402634Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-05T12:40:36.858222779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:40:36.86191062Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:40:36.89699006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:40:36.903376926Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:40:37.164960234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:40:37.165150436Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-05T12:40:40.49838773Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-05T12:40:42.164343369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:40:43.831791701Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=info ts=2026-05-05T12:40:44.084488524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:40:44.087318173Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:40:47.164682161Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:40:47.164720693Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:40:47.164729947Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:40:47.164743217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:40:47.164917643Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-05T12:40:47.164933436Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-05T12:40:48.329841457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:37194"
ts=2026-05-05T12:40:49.21363362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:40:50.498831351Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-05-05T12:40:52.164918253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:40:53.831343018Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-05T12:40:54.714710684Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:40:56.138530953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:40:56.147856627Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:40:56.81564209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:40:56.823108902Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:40:56.823138877Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:40:56.82318912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:40:56.830306019Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:40:57.164109108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:40:57.168450944Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:40:57.168479848Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=info ts=2026-05-05T12:40:58.449146815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:40:58.45441211Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:40:59.089257994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:40:59.092314034Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:40:59.092340411Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:40:59.092365634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:40:59.094527511Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:40:59.32926474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:40:59.332830268Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
ts=2026-05-05T12:40:59.342540789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:54260"
level=info ts=2026-05-05T12:41:00.270491568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:41:00.274315911Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:41:00.277387313Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:41:00.498877643Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-05T12:41:01.574647118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:41:01.583045789Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:41:02.164363669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:41:02.197523796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:41:02.200627046Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:41:03.831920725Z caller=checkpoint.go:372 msg="writing series" size="558 kB"
level=info ts=2026-05-05T12:41:05.587625001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:41:05.593423014Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:41:07.093836157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:41:07.096903196Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=warn ts=2026-05-05T12:41:07.164663131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:41:07.164745326Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-05T12:41:07.956759315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:41:07.959534084Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:41:08.085605043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="399 kB" avg_comp="399 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:41:08.089604604Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:41:10.321382575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:41:10.324336566Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=debug ts=2026-05-05T12:41:10.497973576Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T12:41:12.165125694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:41:12.348150982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:41:12.351255236Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:41:12.700409684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:41:12.708742539Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:41:13.831875922Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-05T12:41:13.831901268Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T12:41:15.772489716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:41:15.776520275Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:41:16.481740704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:41:16.485931254Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:41:17.143548394Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:41:17.144597787Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-05T12:41:17.165001158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:41:17.165007024Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:41:17.165043768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:41:17.165066566Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:41:17.165076584Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
ts=2026-05-05T12:41:19.216662951Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:41:20.501677161Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-05T12:41:20.501701149Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=2
level=info ts=2026-05-05T12:41:21.068014329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:41:21.071385366Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T12:41:22.165095155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:41:22.349872492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:41:22.357123206Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:41:23.185595171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:41:23.188704344Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:41:23.831328457Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T12:41:24.227447768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:41:24.230564202Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:41:25.825093871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:41:25.827640558Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:41:26.182693865Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:41:26.188678013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:41:26.191430598Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:41:27.164863709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:41:27.229647458Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-05T12:41:27.229678944Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=2
level=debug ts=2026-05-05T12:41:30.498413886Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-05T12:41:32.165168259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:41:32.526173505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:41:32.534164817Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:41:33.832170227Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-05T12:41:35.942709625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:41:37.164514965Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-05T12:41:37.164543503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:41:38.828519765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:41:38.833293479Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=debug ts=2026-05-05T12:41:40.498786439Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-05T12:41:41.781108237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:41:41.784621989Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:41:42.165120004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:41:43.831769022Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T12:41:44.915759704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:41:44.918680841Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:41:45.989830082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:41:45.993585813Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:41:45.993610655Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:41:45.993637244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:41:45.99678728Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:41:47.164310855Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:41:47.164342662Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:41:47.164350908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:41:47.164395528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:41:47.16451451Z caller=checkpoint.go:372 msg="writing series" size="5.0 kB"
level=info ts=2026-05-05T12:41:48.282496116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:41:48.29083711Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:41:49.176178075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:41:49.183862274Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T12:41:49.220156367Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:41:50.100951521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:41:50.104457273Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:41:50.499713915Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:41:50.499747614Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-05T12:41:52.164955045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:41:52.746417124Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:41:53.832179246Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-05T12:41:55.901538519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:41:55.907005288Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-05T12:41:56.30285926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:39910"
level=info ts=2026-05-05T12:41:57.031297369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:41:57.034076883Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-05T12:41:57.164721927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:41:57.165255094Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=info ts=2026-05-05T12:41:58.76415336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:41:58.76727844Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:41:59.158462019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:41:59.166203161Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:41:59.166228419Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:41:59.16626509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:41:59.175433886Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:41:59.544612165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:41:59.548371805Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:42:00.498997997Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-05T12:42:02.164330428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:42:03.158225379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:42:03.16208694Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:42:03.831875886Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:42:04.711272323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:42:04.714038678Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:42:07.072966323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:42:07.076314081Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:42:07.076332563Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:42:07.076362725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:42:07.079117106Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:42:07.16411148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:42:07.165936128Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:42:07.165956222Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-05T12:42:09.854297892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:42:09.857026964Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:42:10.509622085Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-05T12:42:10.509646335Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-05-05T12:42:11.242291685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:42:11.244925041Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T12:42:12.164843604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:42:13.831432586Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-05T12:42:14.42411307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:42:14.427146528Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:42:15.552175129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:42:15.563239181Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:42:17.004632799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:42:17.010364465Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:42:17.010390573Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:42:17.010416145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:42:17.015720131Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:42:17.14345689Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:42:17.1445252Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:42:17.164929805Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:42:17.164952061Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:42:17.164960176Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:42:17.164997811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:42:17.165043749Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
level=info ts=2026-05-05T12:42:18.765433817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:42:18.767934555Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-05T12:42:19.223034677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:42:20.498712809Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-05T12:42:20.898015169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:42:20.900792177Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:42:21.350473247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:42:21.354042763Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:42:21.754884166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:42:21.75774628Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:42:22.164671694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:42:23.07264788Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:42:23.83190534Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-05T12:42:25.655979538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:42:25.658783911Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:42:25.989729324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:42:25.994307547Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:42:26.013384199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:42:26.015803043Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T12:42:27.164120157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:42:27.165303463Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-05T12:42:29.588419167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:42:29.594845197Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:42:29.594866333Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:42:29.594892543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:42:29.601823949Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:42:30.49848221Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=info ts=2026-05-05T12:42:31.661459591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:42:31.66881318Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:42:32.164965736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:42:32.272239029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:42:32.278028261Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:42:32.454698201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:42:32.457842956Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:42:33.831170549Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-05T12:42:34.755607053Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:42:37.164196061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:42:37.165446474Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-05-05T12:42:40.4984447Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-05T12:42:40.727002979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:42:40.735502551Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:42:41.417758191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:42:41.42120652Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:42:41.465417777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:42:41.468792514Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:42:42.164224335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:42:43.113006203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:42:43.115653772Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:42:43.832157288Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T12:42:43.832182007Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-05T12:42:44.274234085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:42:44.282293264Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:42:47.164172733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:42:47.164216462Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:42:47.164230551Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:42:47.164309195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:42:47.16562642Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
level=info ts=2026-05-05T12:42:48.868543223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:42:48.876686593Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-05T12:42:49.225112643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:42:50.500659015Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:42:50.500772485Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
level=info ts=2026-05-05T12:42:50.93101663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:42:50.934581077Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:42:51.851020763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:42:51.854337177Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T12:42:52.164591897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:42:53.831365415Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-05T12:42:54.750964377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:42:54.753857257Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:42:55.841518812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:42:55.845357923Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:42:56.160444701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:42:56.164541824Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=warn ts=2026-05-05T12:42:57.164977862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:42:57.165017004Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-05T12:42:58.413939349Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:43:00.498235357Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-05-05T12:43:02.164768331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:43:02.225883894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:43:02.228660872Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:43:03.187423571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:43:03.191436942Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:43:03.831141346Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-05T12:43:07.164867583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:43:07.164986554Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=info ts=2026-05-05T12:43:07.409766294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:43:07.416314978Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:43:08.970444877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:43:08.973663111Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:43:08.973687846Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:43:08.973712964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:43:08.97582111Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:43:10.206959939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:43:10.210479558Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:43:10.476253622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:43:10.478822648Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T12:43:10.4980811Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=info ts=2026-05-05T12:43:11.03365632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:43:11.036166811Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:43:12.164614044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:43:12.375659233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:43:12.378857231Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:43:13.831827705Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=debug ts=2026-05-05T12:43:14.459499118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:43:14.461123797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T12:43:14.869380367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:43:14.875626877Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:43:16.972335667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:43:16.974823525Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:43:17.143935236Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:43:17.143948467Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:43:17.164325439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:43:17.164352057Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:43:17.164364756Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:43:17.164362295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:43:17.164549652Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
ts=2026-05-05T12:43:18.341165105Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35804"
ts=2026-05-05T12:43:19.22769646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:43:20.499046226Z caller=checkpoint.go:372 msg="writing series" size="479 kB"
level=debug ts=2026-05-05T12:43:20.499093759Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-05T12:43:22.164972906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:43:23.29275924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:43:23.300761715Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:43:23.812950216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:43:23.816173936Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:43:23.831267844Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T12:43:25.48010502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:43:25.483687216Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:43:25.594030544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:43:25.596609752Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:43:27.078364701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:43:27.081113514Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-05T12:43:27.164959416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:43:27.166961281Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:43:27.16697939Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=debug ts=2026-05-05T12:43:30.498732935Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-05T12:43:31.126715175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.2 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:43:31.130452207Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:43:32.164397933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:43:32.555461537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:43:32.562445273Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:43:32.903818077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:43:32.906671612Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:43:33.832083342Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-05T12:43:34.169350233Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:43:35.002733279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:43:35.006192225Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:43:36.942793975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:43:36.94554406Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T12:43:37.164692771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:43:37.164937726Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-05T12:43:38.882866819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:43:38.886480647Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:43:40.018671004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:43:40.02841709Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:43:40.498378801Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T12:43:42.165259702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:43:43.833049936Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-05-05T12:43:43.933224987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="548 B" avg_comp="548 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:43:43.936132189Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:43:46.467844888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:43:46.472530626Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:43:47.164354323Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:43:47.164384835Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:43:47.164392815Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:43:47.164364559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:43:47.164607949Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-05T12:43:47.544541844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:43:47.547219308Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
ts=2026-05-05T12:43:48.342538686Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:46042"
ts=2026-05-05T12:43:49.230408574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:43:50.49893443Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:43:50.498960829Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-05T12:43:52.165146206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:43:53.831682785Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-05-05T12:43:54.128315615Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:43:54.278981546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:43:54.281782466Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:43:54.973439659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:43:54.975934462Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-05T12:43:56.313377735Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34748"
level=info ts=2026-05-05T12:43:57.103410917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:43:57.111164108Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T12:43:57.16417345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:43:57.178685143Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:43:57.178708348Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=info ts=2026-05-05T12:43:59.812279841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:43:59.814967744Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:44:00.498746059Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=debug ts=2026-05-05T12:44:00.921259028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:44:02.164270779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:44:03.831837409Z caller=checkpoint.go:372 msg="writing series" size="960 kB"
level=debug ts=2026-05-05T12:44:03.831862979Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-05T12:44:05.262867341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:44:05.26627338Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=warn ts=2026-05-05T12:44:07.164359907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:44:07.167576028Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-05T12:44:07.167592243Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=1
level=info ts=2026-05-05T12:44:07.580664334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:44:07.583418641Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:44:07.69768289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:44:07.70061353Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:44:10.498310114Z caller=checkpoint.go:372 msg="writing series" size="895 B"
level=info ts=2026-05-05T12:44:11.279386965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:44:11.287738658Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:44:12.16512523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:44:13.157968797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:44:13.161146526Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:44:13.161176252Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:44:13.161203693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:44:13.164308658Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:44:13.832150384Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:44:17.119836014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:44:17.128027084Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:44:17.143997671Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:44:17.144073763Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:44:17.164416834Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:44:17.1644759Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:44:17.164473994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:44:17.164489484Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:44:17.169455512Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-05T12:44:17.169483649Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=3
level=info ts=2026-05-05T12:44:17.717393478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:44:17.720744188Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:44:19.216419624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:44:19.224145869Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
ts=2026-05-05T12:44:19.231931971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:44:20.498944045Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=info ts=2026-05-05T12:44:21.048510228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:44:21.051473368Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:44:21.083517875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:44:21.086981567Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:44:21.087007507Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:44:21.087035709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:44:21.090097755Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:44:22.165016864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:44:22.789350999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:44:22.792805519Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:44:22.797857734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:44:22.80052541Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:44:23.06223981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:44:23.065692346Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:44:23.831880906Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
ts=2026-05-05T12:44:26.315263352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:35404"
level=warn ts=2026-05-05T12:44:27.164358413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:44:27.164421489Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-05T12:44:27.333026248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:44:27.342183668Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:44:29.488274984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:44:29.491428338Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:44:29.635965389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:44:29.639022883Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:44:30.349699608Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:44:30.499339637Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:44:30.499362307Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=4
level=info ts=2026-05-05T12:44:31.157140357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:44:31.161235651Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:44:31.345783105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:44:31.348491876Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T12:44:32.164992118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:44:33.831133498Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-05T12:44:36.955102561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:44:36.958100049Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T12:44:37.164975208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:44:37.165033343Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-05T12:44:40.321656053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:44:40.330257944Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:44:40.498181757Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-05T12:44:42.165106796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:44:42.47120366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:44:42.475449248Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:44:42.715286963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:44:42.718856621Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=debug ts=2026-05-05T12:44:43.831463111Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=debug ts=2026-05-05T12:44:47.164899916Z caller=checkpoint.go:384 msg="flushing series" totalSize="641 kB" series=4
level=info ts=2026-05-05T12:44:47.164912995Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:44:47.164936693Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:44:47.164946057Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:44:47.164964829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:44:47.172151335Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-05T12:44:47.173533187Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009165393s
ts=2026-05-05T12:44:48.348756716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:40596"
ts=2026-05-05T12:44:49.234719635Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:44:50.022357133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:44:50.026825831Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:44:50.992923919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:44:50.996500814Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:44:52.164792197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:44:54.450366458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:44:57.164602622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:44:58.054610236Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:45:00.930734878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="627 B" avg_comp="627 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:45:00.935210133Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:45:01.05757144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:45:01.0603524Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:45:01.398330175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:45:01.401300422Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:45:01.93387563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:45:01.936677992Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T12:45:02.164559684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:45:02.373345614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:45:02.376762843Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:45:03.93287008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:45:03.936361051Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:45:04.523181066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:45:04.526584156Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:45:05.471155842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:45:05.478922795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:45:06.493362242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:45:06.496847129Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:45:07.164847281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:45:07.36247104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:45:07.366628737Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:45:07.366654477Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:45:07.366682216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:45:07.368840393Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:45:08.132168971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:45:08.135347108Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:45:08.730199085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:45:08.737731772Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:45:10.933735685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:45:10.941666766Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:45:12.16429282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:45:15.055117032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:45:15.064545235Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:45:17.141483778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:45:17.143437032Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:45:17.143904281Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=error ts=2026-05-05T12:45:17.14445359Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:45:17.144673954Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-05T12:45:17.144737439Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975538 n=1
level=debug ts=2026-05-05T12:45:17.144797142Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975538 n=1
level=info ts=2026-05-05T12:45:17.168043992Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-05T12:45:17.168034286Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-05T12:45:17.168102171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:45:17.168113985Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:45:17.168116455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:45:17.168261125Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-05T12:45:17.168269741Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:45:17.168292825Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.257µs success=true
level=debug ts=2026-05-05T12:45:17.168562137Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-05T12:45:18.397206863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:45:18.399898273Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-05T12:45:19.236748449Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:45:20.503352565Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=info ts=2026-05-05T12:45:21.044486942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:45:21.047864364Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T12:45:22.164470858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:45:23.835466508Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=debug ts=2026-05-05T12:45:24.080280518Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:45:24.450457964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:45:24.456910248Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:45:26.847163275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:45:26.851670404Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:45:27.041267033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:45:27.043888942Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-05T12:45:27.164720877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:45:27.169118895Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-05-05T12:45:27.16913586Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-05T12:45:29.369470056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:55168"
level=debug ts=2026-05-05T12:45:30.502293169Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-05T12:45:32.164889571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:45:33.474181196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:45:33.477571798Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:45:33.477593001Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:45:33.477615989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:45:33.479820075Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:45:33.83603002Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-05T12:45:37.164764722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:45:37.169255349Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-05T12:45:37.304243206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:45:37.306778796Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:45:37.306799106Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:45:37.306823388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:45:37.309305256Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=debug ts=2026-05-05T12:45:40.502209562Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=info ts=2026-05-05T12:45:41.173977151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:45:41.177045035Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=warn ts=2026-05-05T12:45:42.164130165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:45:43.202612715Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:45:43.835956112Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=debug ts=2026-05-05T12:45:43.835979152Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T12:45:45.647243085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:45:45.65004386Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:45:45.650081908Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:45:45.650108381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:45:45.652078064Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:45:45.679180947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:45:45.682114161Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:45:47.164909878Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:45:47.16494725Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:45:47.164956295Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:45:47.164973693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:45:47.169338425Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-05T12:45:48.205970211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:45:48.208926892Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:45:49.076357895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:45:49.079734255Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-05T12:45:49.238957293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:45:49.67610933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:45:49.678649306Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:45:49.678668443Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:45:49.678689948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:45:49.680669537Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:45:50.225953538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:45:50.229297301Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:45:50.502125644Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=warn ts=2026-05-05T12:45:52.164474439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:45:52.402344931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:45:52.405338393Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:45:52.4214022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:45:52.424528079Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-05T12:45:52.42454565Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:45:52.424572345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:45:52.427326011Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:45:53.390194109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:45:53.393148755Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:45:53.665957246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:45:53.668752313Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=debug ts=2026-05-05T12:45:53.835382206Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:45:55.58891853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:45:55.595579279Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-05T12:45:56.32372479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:33988"
level=warn ts=2026-05-05T12:45:57.164196461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:45:57.171020114Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:45:57.171040513Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=info ts=2026-05-05T12:45:57.17949184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:45:57.185756337Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:45:57.659818279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:45:57.666385241Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:45:57.731471711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:45:57.734549803Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:45:57.866114255Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:45:59.713774519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:45:59.720540645Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:45:59.720570387Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:45:59.720621876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:45:59.727067927Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:45:59.731124487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:45:59.734101054Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:46:00.503010907Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-05T12:46:02.16501626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:46:03.082112845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:46:03.08460145Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:46:03.83629399Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=info ts=2026-05-05T12:46:03.949333652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:46:03.95260369Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:46:05.589631439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:46:05.592485054Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T12:46:07.164101287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:46:07.169464167Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-05T12:46:07.636784607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:46:07.639545905Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:46:07.639565963Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:46:07.639591789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:46:07.641703168Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:46:08.452495791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:46:08.455151498Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:46:09.27847768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:46:09.282282602Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:46:10.502503853Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T12:46:12.164958015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:46:13.835896585Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-05T12:46:13.835924254Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-05T12:46:16.726450501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:46:16.729447404Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:46:16.729474765Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:46:16.729504298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:46:16.731730255Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:46:17.14387849Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:46:17.144142008Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:46:17.164480458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:46:17.164504048Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:46:17.164512735Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:46:17.164517921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:46:17.168704124Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-05T12:46:17.66521288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:46:17.673247376Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:46:18.186095951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:46:18.188673128Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:46:18.304437855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:46:18.354208063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56556"
level=info ts=2026-05-05T12:46:19.181914137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:46:19.188878541Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-05T12:46:19.241349956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:46:20.502806023Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-05T12:46:20.50283022Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-05-05T12:46:21.893482073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:46:21.896265882Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=warn ts=2026-05-05T12:46:22.164686738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:46:23.835424891Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T12:46:25.724188937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:46:25.726800688Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-05T12:46:26.326327967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:42738"
level=warn ts=2026-05-05T12:46:27.164191679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:46:27.244656544Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-05T12:46:27.244688328Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=2
level=debug ts=2026-05-05T12:46:28.879014675Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:46:29.085588925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:46:29.089111405Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=debug ts=2026-05-05T12:46:30.503156086Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-05T12:46:31.804966082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:46:31.808194211Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T12:46:32.165214007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:46:33.836126099Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-05T12:46:34.922135096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:46:34.925786554Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:46:34.925815329Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:46:34.925842258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:46:34.928418413Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:46:36.776096646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:46:36.781128003Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:46:37.164670681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:46:37.168858961Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-05T12:46:37.835916976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:46:37.843845599Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:46:40.195417387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:46:40.198662437Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:46:40.502568597Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-05T12:46:40.585858556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:46:40.588651116Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-05T12:46:40.58867399Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:46:40.588704306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:46:40.591007854Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:46:41.499310648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:46:41.503596819Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:46:42.16490897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:46:42.472970646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:46:42.475793092Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:46:43.835285343Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T12:46:44.212496332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:46:44.215586579Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:46:45.461698597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:46:45.472783741Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:46:45.636137503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:46:45.643802679Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:46:47.164878159Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:46:47.164910373Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:46:47.164919256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:46:47.16492407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:46:47.169202194Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-05T12:46:47.47549174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:46:47.47837898Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-05T12:46:48.355477989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36310"
level=info ts=2026-05-05T12:46:48.595944617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:46:48.603785646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:46:49.104277524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:46:49.107837394Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:46:49.107859503Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:46:49.107884591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:46:49.111021906Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-05T12:46:49.24457788Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:46:49.659267911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:46:49.662843864Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:46:50.50341563Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:46:50.503440857Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-05T12:46:52.16480737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:46:53.210576587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:46:53.214228734Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:46:53.836555251Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
ts=2026-05-05T12:46:56.328862234Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:32864"
level=info ts=2026-05-05T12:46:56.617437928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:46:56.620965075Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:46:56.816423471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:46:56.819315231Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T12:46:57.164566463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:46:57.169036872Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=debug ts=2026-05-05T12:46:57.668946573Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:46:58.141785068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:46:58.146238975Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:46:58.405443476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:46:58.408829856Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:46:59.555560539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:46:59.563207219Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:47:00.502951953Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-05T12:47:02.164395129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:47:03.836279567Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:47:05.06883077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:47:05.072023586Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:47:06.538830887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:47:06.542102036Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T12:47:07.164826797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:47:07.169509241Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-05T12:47:07.169529965Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-05T12:47:08.99508586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:47:08.997922608Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:47:08.997948384Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:47:08.997979467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:47:09.000208041Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=debug ts=2026-05-05T12:47:10.515491609Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-05T12:47:10.515526703Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=1
level=debug ts=2026-05-05T12:47:11.022224155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:47:11.023841999Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T12:47:11.51278037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:47:11.51580082Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:47:12.165191666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:47:13.534936811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:47:13.538000399Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:47:13.835819387Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-05T12:47:17.143883988Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:47:17.143890967Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:47:17.164260889Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:47:17.164288245Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:47:17.164296913Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:47:17.16428764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:47:17.169492595Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
ts=2026-05-05T12:47:19.247344578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:47:19.950554838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:47:19.955185892Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=debug ts=2026-05-05T12:47:20.501943725Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:47:22.164650086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:47:22.918100176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:47:22.924793195Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:47:23.835677955Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-05T12:47:27.010823516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:47:27.014940459Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T12:47:27.164745412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:47:27.168927734Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-05T12:47:27.271181052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:47:27.273950742Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:47:28.242753065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:47:28.245444205Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:47:28.557383599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:47:28.559930477Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:47:28.966047686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:47:28.969257551Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:47:29.118702335Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:47:29.123689991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:47:29.131351852Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:47:29.324021234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:47:29.326539134Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-05T12:47:29.381910332Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:35148"
level=debug ts=2026-05-05T12:47:30.502570451Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-05T12:47:31.669571307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:47:31.672210998Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T12:47:32.164695086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:47:33.017421634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:47:33.020888332Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:47:33.835481146Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-05T12:47:34.707838994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:47:34.714988758Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:47:37.165042885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:47:37.169410637Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-05-05T12:47:38.494717259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:47:38.498785902Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:47:40.001099484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:47:40.004579359Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:47:40.502231895Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-05T12:47:40.502263989Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-05T12:47:42.164941281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:47:43.255509072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:47:43.262786853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:47:43.262808015Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:47:43.26284719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:47:43.27007732Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:47:43.836187652Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-05T12:47:47.164675055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:47:47.16468274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:47:47.164710197Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:47:47.164720834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:47:47.169137802Z caller=checkpoint.go:372 msg="writing series" size="625 kB"
level=info ts=2026-05-05T12:47:48.335185814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:47:48.337627117Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:47:48.513828904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:47:49.250385731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:47:50.503504081Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:47:50.503528557Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=warn ts=2026-05-05T12:47:52.164122275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:47:53.835587332Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-05T12:47:55.236269861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:47:55.239755154Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:47:55.818355507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:47:55.821560741Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:47:56.033278727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:47:56.042796165Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:47:56.174336455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:47:56.178262373Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:47:56.178288754Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:47:56.178317944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:47:56.181682934Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:47:56.333808306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:40376"
level=warn ts=2026-05-05T12:47:57.164138965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:47:57.169312834Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-05T12:47:58.642629727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:47:58.651827987Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:48:00.502713376Z caller=checkpoint.go:372 msg="writing series" size="281 kB"
level=warn ts=2026-05-05T12:48:02.165120141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:48:03.390038513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:48:03.392708341Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:48:03.83598929Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-05T12:48:04.065709351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:48:04.068737019Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:48:04.729805904Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:48:06.675242442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:48:06.679544958Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:48:07.164899948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:48:07.16927097Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-05T12:48:07.181598018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:48:07.184451921Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:48:08.807943522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:48:08.811443525Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:48:09.917618492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:48:09.921661501Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:48:10.502715134Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-05T12:48:12.164844209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:48:12.646384193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:48:12.653163346Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:48:13.835945244Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-05T12:48:14.851264791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:48:14.859584965Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:48:14.859610576Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:48:14.85963984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:48:14.867140025Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:48:16.789271628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:48:16.79253781Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:48:17.143594031Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:48:17.144723591Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:48:17.164112136Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:48:17.164140741Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:48:17.164152359Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:48:17.164171364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:48:17.169450864Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
ts=2026-05-05T12:48:19.252850158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:48:19.447569042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:48:19.4502273Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:48:19.992911679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:48:20.000647737Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:48:20.503186362Z caller=checkpoint.go:372 msg="writing series" size="556 kB"
level=debug ts=2026-05-05T12:48:20.50320999Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=9
level=info ts=2026-05-05T12:48:21.544518027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:48:21.547868693Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:48:21.616982665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:48:21.626240354Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:48:22.165144603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:48:23.647984674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:48:23.652009927Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T12:48:23.835671933Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-05T12:48:26.335834028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:44132"
level=info ts=2026-05-05T12:48:26.341116288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:48:26.344802701Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:48:26.815445648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:48:26.818337852Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-05T12:48:27.164369576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:48:27.172119144Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:48:27.172144271Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-05-05T12:48:28.435613771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:48:28.438283133Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-05T12:48:29.387929586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:41424"
level=info ts=2026-05-05T12:48:29.663265186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:48:29.671198653Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:48:30.502460348Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-05T12:48:32.164622249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:48:32.47081582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:48:32.473468255Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:48:33.835544984Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=debug ts=2026-05-05T12:48:34.386202535Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:48:37.164102634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:48:37.169318697Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-05T12:48:38.046946052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:48:38.050857883Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:48:38.050882361Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:48:38.050911494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:48:38.053889656Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:48:40.502339467Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-05T12:48:42.16501416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:48:43.836272021Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-05-05T12:48:44.292260679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:48:44.298668301Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:48:44.298690166Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:48:44.298717008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:48:44.305160864Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:48:44.429641016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:48:44.432595822Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:48:44.835797259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:48:44.839326465Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:48:46.592596718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="548 B" avg_comp="548 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:48:46.595190798Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:48:47.1649225Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:48:47.164932954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:48:47.164950586Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:48:47.164959474Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:48:47.169197946Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-05T12:48:49.255955682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:48:49.543368904Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:48:50.503113919Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:48:50.503136701Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-05T12:48:52.165040532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:48:52.561255179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:48:52.564094516Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:48:52.771722445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:48:52.774881785Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:48:53.836174488Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:48:55.035951344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:48:55.043301604Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:48:55.676671238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:48:55.683100992Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:48:56.292421673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:48:56.300035112Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:48:56.300082944Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:48:56.30011256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:48:56.307273557Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:48:57.164375013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:48:57.186805156Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:48:57.186831941Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=info ts=2026-05-05T12:48:58.138317766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:48:58.140715178Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:48:59.061549515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:48:59.065939549Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:48:59.072970776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:48:59.075283967Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:49:00.43890082Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:49:00.502085872Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-05-05T12:49:02.165218394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:49:02.649650168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:49:02.656498228Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:49:03.837047189Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=debug ts=2026-05-05T12:49:03.837100879Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-05T12:49:03.962684709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:49:03.97018525Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:49:07.164197034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:49:07.173569621Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-05-05T12:49:07.173596263Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=1
level=info ts=2026-05-05T12:49:08.737351136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:49:08.740266179Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:49:10.501945192Z caller=checkpoint.go:372 msg="writing series" size="895 B"
level=info ts=2026-05-05T12:49:11.353737421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:49:11.35771663Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:49:12.164390811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:49:12.398968589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:49:12.401934242Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:49:12.832702492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:49:12.835989814Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:49:13.410809966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:49:13.419275774Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:49:13.835969211Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:49:15.10066236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:49:15.103676372Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:49:15.352673891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:49:15.355794426Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:49:17.143882779Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:49:17.144087012Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:49:17.164405863Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:49:17.164430435Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:49:17.164439355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:49:17.164492892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:49:17.170469452Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-05T12:49:17.170489797Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=3
ts=2026-05-05T12:49:18.369384451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:58648"
level=info ts=2026-05-05T12:49:18.623372113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:49:18.629851353Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-05T12:49:19.25820558Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:49:20.502639983Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-05T12:49:21.128338871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:49:21.131420631Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:49:22.164740892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:49:22.635266846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:49:22.638142076Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:49:23.836487991Z caller=checkpoint.go:372 msg="writing series" size="651 kB"
ts=2026-05-05T12:49:26.341437607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:46298"
level=warn ts=2026-05-05T12:49:27.164084693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:49:27.169294689Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-05T12:49:27.445391408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="627 B" avg_comp="627 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:49:27.450195665Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:49:28.571030826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:49:28.57503302Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:49:29.393710937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:38374"
level=info ts=2026-05-05T12:49:29.9866963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:49:29.993434316Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:49:30.504102304Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:49:30.504124866Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=info ts=2026-05-05T12:49:31.258604258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:49:31.261411431Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:49:31.754522953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:49:31.757888627Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:49:31.757909798Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:49:31.757933427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:49:31.760292481Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:49:31.905163204Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:49:32.164461382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:49:33.836116669Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-05-05T12:49:37.164993101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:49:37.169254487Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-05T12:49:39.471472087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:49:39.474450867Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:49:40.502526661Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-05T12:49:40.722992512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:49:40.725652813Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:49:42.164623936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:49:43.794111708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:49:43.797941147Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:49:43.836211713Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-05T12:49:45.892263182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:49:45.900883478Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:49:46.161733586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:49:46.164418074Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:49:46.50571272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:49:46.512351149Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:49:47.164196102Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:49:47.164226736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:49:47.164234069Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:49:47.164249898Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:49:47.169341679Z caller=checkpoint.go:384 msg="flushing series" totalSize="698 kB" series=4
level=info ts=2026-05-05T12:49:47.176224952Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-05T12:49:47.177580805Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009106214s
ts=2026-05-05T12:49:49.260137893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:49:49.261088647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:49:49.264467201Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:49:49.354833587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-05T12:49:52.164421615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:49:53.524309903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:49:53.527382953Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:49:54.247534408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:49:54.251031664Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:49:55.18971375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:49:55.196502345Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:49:57.165102469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:49:59.023661282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:49:59.033042075Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:50:00.60665386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:50:00.609428056Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T12:50:00.937503731Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:50:02.164738459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:50:06.339812919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:50:06.344993152Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T12:50:07.164575944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:50:09.139622336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:50:09.142995872Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:50:09.432879017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:50:09.436038629Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:50:10.118027578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:50:10.122490062Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:50:10.398649618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:50:10.40130882Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:50:11.120614327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:50:11.124790328Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:50:12.164342773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:50:12.214499161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:50:12.217117501Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:50:13.694268506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:50:13.70022821Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:50:13.78270712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:50:13.785849359Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:50:17.143875976Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:50:17.143903432Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:50:17.164241124Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-05T12:50:17.164251669Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-05T12:50:17.164270214Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:50:17.164279284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:50:17.164297114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:50:17.164428782Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-05T12:50:17.164438677Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:50:17.16446223Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.452µs success=true
level=debug ts=2026-05-05T12:50:17.16480005Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-05T12:50:19.262334179Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:50:19.584671699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:50:19.587371072Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=debug ts=2026-05-05T12:50:20.499888932Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=warn ts=2026-05-05T12:50:22.164358641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:50:23.548492117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:50:23.55150021Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:50:23.83242871Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-05T12:50:26.536650356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:50:26.539663986Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:50:26.997127309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:50:27.004395579Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:50:27.164575476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:50:27.166030177Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=debug ts=2026-05-05T12:50:27.166044321Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-05T12:50:27.365933954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:50:27.373941632Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:50:28.24188409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:50:30.498385864Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-05T12:50:32.164622283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:50:33.832072242Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-05T12:50:34.609684484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:50:34.613109567Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:50:35.076318249Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:50:36.398696098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:50:36.401661832Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:50:36.991749419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:50:36.995241194Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:50:37.164820491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:50:37.164993972Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-05T12:50:39.825292133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:50:39.828109643Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:50:40.499160958Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-05T12:50:42.16473512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:50:43.440103708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:50:43.443623546Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T12:50:43.832351148Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=debug ts=2026-05-05T12:50:43.832373953Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T12:50:47.164201743Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:50:47.164254213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:50:47.164263144Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:50:47.164270117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:50:47.16545914Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
ts=2026-05-05T12:50:49.264610582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:50:50.498672634Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=info ts=2026-05-05T12:50:51.28440622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="114 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:50:51.292012169Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:50:52.164330678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:50:52.352101703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:50:52.358996054Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:50:53.154449844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:50:53.157350154Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:50:53.666622773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:50:53.670108881Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:50:53.831609616Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:50:56.138709584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:50:56.142553005Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:50:56.142580518Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:50:56.142606115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:50:56.145978676Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:50:56.265207977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:50:56.268583258Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
ts=2026-05-05T12:50:56.348959358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:45316"
level=info ts=2026-05-05T12:50:56.44046393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:50:56.446657722Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:50:57.040262903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:50:57.043347119Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:50:57.043368018Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:50:57.043398121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:50:57.046029259Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:50:57.165192952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:50:57.167639347Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:50:57.167665375Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=info ts=2026-05-05T12:50:58.304108428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:50:58.306774364Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:50:58.970259593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:50:58.973836791Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:50:59.402800846Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:43216"
level=debug ts=2026-05-05T12:51:00.446968538Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:51:00.499185519Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-05T12:51:02.164653039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:51:03.831920018Z caller=checkpoint.go:372 msg="writing series" size="492 kB"
level=info ts=2026-05-05T12:51:03.968106078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:51:03.97086129Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:51:05.704564289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:51:05.707207838Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=info ts=2026-05-05T12:51:06.298942289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:51:06.302144939Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:51:06.957602282Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:51:06.959175136Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-05T12:51:07.164590858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:51:07.164761472Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-05T12:51:07.545666446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:51:07.548227457Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:51:09.255112823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:51:09.258111671Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:51:10.498579319Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T12:51:12.16489185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:51:13.073990688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:51:13.077591091Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:51:13.831925029Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-05T12:51:13.83194843Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T12:51:15.928370149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:51:15.932261408Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:51:17.143787001Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:51:17.14491413Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-05T12:51:17.164254336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:51:17.164252518Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:51:17.164278787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:51:17.16428762Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:51:17.165415519Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
ts=2026-05-05T12:51:19.265982392Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:51:19.512103129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:51:19.519548727Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:51:20.104359458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:51:20.108097047Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:51:20.501517647Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-05T12:51:20.501541338Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=2
level=info ts=2026-05-05T12:51:21.669683608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:51:21.672320635Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T12:51:22.164649345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:51:23.831749553Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T12:51:24.189875363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:51:24.19764469Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:51:24.761756802Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:51:24.761801153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="405 kB" avg_comp="405 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" idle=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:51:24.771405538Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=warn ts=2026-05-05T12:51:27.165020324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:51:27.236898615Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-05T12:51:27.236934634Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=2
level=info ts=2026-05-05T12:51:28.938439683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:51:28.941911336Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-05T12:51:29.405494331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:58724"
level=debug ts=2026-05-05T12:51:30.499171948Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=warn ts=2026-05-05T12:51:32.164903957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:51:33.46159349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:51:33.466367462Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:51:33.831645359Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-05T12:51:33.843672391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:51:33.846518355Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:51:37.164567493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:51:37.164753291Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-05T12:51:37.196735974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:51:37.202283285Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=debug ts=2026-05-05T12:51:40.498670664Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-05T12:51:42.165114795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:51:43.80723666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:51:43.810406833Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:51:43.831591824Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T12:51:44.806161807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:51:44.81120134Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:51:45.288677818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:51:45.295172676Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:51:46.764903657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:51:46.768390581Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:51:46.809523585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:51:46.81728478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:51:46.817688673Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:51:46.820131393Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:51:47.164333761Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:51:47.164349219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:51:47.164370432Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:51:47.164381227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:51:47.165591883Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-05T12:51:48.460489488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:51:48.463359188Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:51:48.46338218Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:51:48.463410071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:51:48.465662199Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:51:48.987077942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:51:48.994943414Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-05T12:51:49.267858199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:51:50.499776896Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:51:50.499801339Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-05T12:51:50.750800344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:51:50.754927668Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T12:51:52.164931101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:51:52.235172336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:51:52.240513373Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:51:52.800801063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:51:52.80351361Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=debug ts=2026-05-05T12:51:53.831752432Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=debug ts=2026-05-05T12:51:53.853674389Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:51:54.662107044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:51:54.664957746Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:51:56.527714446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:51:56.530496562Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T12:51:57.164954273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:51:57.165476516Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=info ts=2026-05-05T12:52:00.019596035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:52:00.022227648Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:52:00.499204731Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-05T12:52:01.840238665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:52:01.842850906Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=warn ts=2026-05-05T12:52:02.165172194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:52:02.672105153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:52:02.674652769Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:52:02.987648548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:52:03.832321812Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:52:04.296821036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:52:04.300239547Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=info ts=2026-05-05T12:52:04.865901225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:52:04.869336913Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-05T12:52:04.869355813Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:52:04.869377628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:52:04.872220172Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:52:05.622371989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:52:05.62871926Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:52:06.33395992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:52:06.341373727Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:52:07.164379295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:52:07.16588978Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-05T12:52:07.165903732Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-05T12:52:10.433697211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:52:10.438173135Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:52:10.510572843Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-05T12:52:10.510597427Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=1
level=info ts=2026-05-05T12:52:10.905189772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:52:10.907953976Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T12:52:12.165086578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:52:13.831745135Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=debug ts=2026-05-05T12:52:17.143865343Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:52:17.145441855Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:52:17.16476458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:52:17.164769622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:52:17.164787383Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:52:17.16479749Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:52:17.164809367Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
level=info ts=2026-05-05T12:52:17.981186268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:52:17.984247499Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:52:17.984267631Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:52:17.984291182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:52:17.986840191Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:52:18.275867862Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-05T12:52:19.270026525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:52:20.498345134Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-05T12:52:20.58877185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:52:20.591419912Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:52:20.591437675Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:52:20.591458335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:52:20.593397837Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:52:20.785027288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:52:20.78818813Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:52:22.16489951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:52:23.832283433Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-05T12:52:25.91908378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:52:25.923679967Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:52:27.164104806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:52:27.165193512Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-05T12:52:28.863392752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:52:28.86614292Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:52:28.866162837Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:52:28.866186445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:52:28.868292283Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:52:30.476549337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:52:30.479685592Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:52:30.498917915Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=info ts=2026-05-05T12:52:32.153166059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:52:32.156658245Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:52:32.164709246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:52:33.052235609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:52:33.058573429Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:52:33.831567098Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-05T12:52:34.130190726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="60 MB" avg_comp="2.9 MB" total_uncomp="497 MB" avg_uncomp="24 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:52:34.138380746Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:52:34.798258353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:52:34.800992129Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:52:35.033725317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:52:35.0362345Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:52:35.036256469Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:52:35.036277961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:52:35.038304875Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T12:52:37.164216066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:52:37.165476785Z caller=checkpoint.go:372 msg="writing series" size="394 kB"
level=info ts=2026-05-05T12:52:38.629794775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:52:38.632415712Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:52:38.632436431Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:52:38.632457769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:52:38.634497098Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:52:39.544197671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:52:39.547586072Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:52:39.917507492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:52:39.920030679Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:52:40.012379627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:52:40.014994234Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:52:40.498326391Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-05T12:52:40.49834607Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-05T12:52:42.16491428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:52:42.385323713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:52:42.38835823Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:52:43.009706537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:52:43.016256278Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:52:43.016276243Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:52:43.016303427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:52:43.022686495Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:52:43.681709637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:52:43.684324784Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:52:43.832153753Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-05T12:52:45.667110568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:52:45.669947958Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:52:46.507498074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:52:46.511103506Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=info ts=2026-05-05T12:52:47.16415466Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:52:47.164185959Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:52:47.164177722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:52:47.164194826Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:52:47.165972741Z caller=checkpoint.go:372 msg="writing series" size="645 kB"
level=info ts=2026-05-05T12:52:48.16334494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:52:48.165969925Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:52:49.081424634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:52:49.084128845Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:52:49.084150914Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:52:49.084175777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:52:49.085993543Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-05T12:52:49.272931775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:52:50.499739567Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-05T12:52:50.49976497Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=info ts=2026-05-05T12:52:50.533831095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:52:50.542440426Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:52:51.211753846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:52:51.214917314Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:52:52.164810572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:52:52.192985168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:52:52.195410597Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:52:52.428044342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:52:52.435633563Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:52:53.2120434Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:52:53.831567875Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-05T12:52:54.697254427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:52:54.69987557Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-05T12:52:54.699894635Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:52:54.699917736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:52:54.701944436Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T12:52:57.164843538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:52:57.164929092Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-05T12:52:57.407292308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:52:57.413961147Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:52:57.420985807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:52:57.423107033Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=debug ts=2026-05-05T12:52:58.926098249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:53:00.499071117Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=info ts=2026-05-05T12:53:01.98732578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:53:01.990046986Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T12:53:02.164594872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:53:02.237812233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:53:02.240213898Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:53:02.589420048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:53:02.591852262Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:53:02.59187571Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:53:02.591897798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:53:02.593890398Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:53:03.832138295Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-05T12:53:07.164984327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:53:07.165108924Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=info ts=2026-05-05T12:53:09.185845257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:53:09.189218284Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:53:10.49877835Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-05T12:53:12.165117065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:53:13.831378152Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-05T12:53:13.967877581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:53:13.97019276Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:53:15.644483709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:53:15.64753638Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:53:15.696582673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:53:15.699961126Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:53:17.143128844Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:53:17.144181271Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:53:17.164568817Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:53:17.16459048Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:53:17.164601691Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:53:17.164622381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:53:17.165757462Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-05T12:53:18.15198781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:53:18.157414703Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-05T12:53:19.276191136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:53:19.536382977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:53:19.539778673Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T12:53:20.498937967Z caller=checkpoint.go:372 msg="writing series" size="454 kB"
level=debug ts=2026-05-05T12:53:20.498961454Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-05T12:53:21.089483226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:53:21.092914994Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:53:22.165210546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:53:23.330292305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:53:23.332870361Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:53:23.83222622Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-05T12:53:25.301728956Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:53:26.076898496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:53:26.079340816Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:53:26.481650596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:53:26.487599598Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:53:27.164080038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:53:27.167420849Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:53:27.167437157Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-05-05T12:53:29.261606414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:53:29.264555705Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=debug ts=2026-05-05T12:53:30.49884786Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-05T12:53:31.900127697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:53:31.904825592Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=warn ts=2026-05-05T12:53:32.165027221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:53:32.271368928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:53:32.277992807Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:53:32.570998631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:53:32.574455723Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:53:33.831823346Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-05T12:53:35.277888715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="548 B" avg_comp="548 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:53:35.28033224Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:53:35.421696942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:53:35.424237648Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:53:35.537644144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:53:35.540219625Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:53:36.487773329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:53:36.493770064Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:53:37.007404283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:53:37.010695513Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:53:37.010718035Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:53:37.010741552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:53:37.015607449Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T12:53:37.164651633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:53:37.165841295Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-05T12:53:38.078303734Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:53:38.90765175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:53:38.910745785Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:53:40.498645096Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:53:40.654119762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f2f1336a61dff054 immediate=false num_chunks=1 total_comp="645 B" avg_comp="645 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:53:40.657091306Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f2f1336a61dff054/19df7ead6f1%3A19df7eafe21%3A1a689f63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-jpdu\"}"
level=error ts=2026-05-05T12:53:40.657116242Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:53:40.657142327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T12:53:40.65957565Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-05T12:53:42.16424507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:53:42.283570213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:53:42.286225311Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:53:43.832619858Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=warn ts=2026-05-05T12:53:47.164743876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:53:47.164740207Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:53:47.164774679Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:53:47.164783149Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:53:47.164820403Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-05T12:53:48.776493703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:53:48.779290183Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-05T12:53:49.278892004Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:53:50.499215492Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:53:50.499245712Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-05T12:53:51.114126021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:53:51.120494706Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=warn ts=2026-05-05T12:53:52.164688999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:53:53.831571671Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:53:55.760043146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:53:55.763180395Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:53:56.999449275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:53:57.002672151Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T12:53:57.164172935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:53:57.182043045Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:53:57.182106202Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=info ts=2026-05-05T12:53:57.208544901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T12:53:57.211560534Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-05T12:53:59.419887149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:48568"
level=info ts=2026-05-05T12:53:59.514658605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:53:59.517470043Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:54:00.414046277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:54:00.420753689Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:54:00.49910381Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-05-05T12:54:00.909927929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:54:02.164306761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:54:03.832667052Z caller=checkpoint.go:372 msg="writing series" size="988 kB"
level=warn ts=2026-05-05T12:54:07.164553279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:54:07.16880269Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-05T12:54:07.1688204Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.0 MB" series=3
level=info ts=2026-05-05T12:54:07.475538857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:54:07.478166176Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:54:10.498850607Z caller=checkpoint.go:372 msg="writing series" size="895 B"
level=info ts=2026-05-05T12:54:10.726541331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:54:10.72993077Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:54:11.424032812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:54:11.431639747Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:54:12.164872136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:54:12.81659613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:54:12.820004049Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:54:13.831577898Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:54:14.47955379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:54:14.482943736Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:54:17.144016958Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:54:17.144352797Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=warn ts=2026-05-05T12:54:17.164888318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:54:17.164897641Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:54:17.164927818Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:54:17.164939212Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:54:17.167783851Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-05T12:54:17.167802172Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=3
level=info ts=2026-05-05T12:54:19.106209853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:54:19.110549303Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-05T12:54:19.281506405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:54:19.783635008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:54:19.786567205Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:54:20.324893304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:54:20.327613203Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:54:20.498294145Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-05-05T12:54:21.48131179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:54:21.48415972Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:54:22.065205195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:54:22.068029236Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=warn ts=2026-05-05T12:54:22.164283846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:54:23.832474707Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=info ts=2026-05-05T12:54:24.981634462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:54:24.984282601Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-05T12:54:26.365655461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:55802"
level=debug ts=2026-05-05T12:54:27.164786294Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-05T12:54:27.164823651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:54:29.03140989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:54:29.035930456Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:54:29.035959479Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:54:29.035985283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:54:29.04041053Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:54:29.423193941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:42440"
level=debug ts=2026-05-05T12:54:30.500255862Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:54:30.500279576Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=4
level=info ts=2026-05-05T12:54:30.545865826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:54:30.553655651Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:54:30.794305355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:54:30.799758815Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=info ts=2026-05-05T12:54:31.859763239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:54:31.866961087Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:54:31.866988272Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:54:31.867030738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:54:31.874652189Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:54:31.956019769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:54:31.963588687Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:54:31.963610672Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=warn ts=2026-05-05T12:54:32.164753481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:54:32.167891776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:54:32.175714787Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:54:32.379049815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:54:32.382264455Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:54:33.127779587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:54:33.130629384Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:54:33.831620398Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=info ts=2026-05-05T12:54:36.341687888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:54:36.344457903Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:54:36.781914191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T12:54:36.784361949Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-05T12:54:36.862543785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:54:37.164640669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:54:37.164855269Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=debug ts=2026-05-05T12:54:38.871323054Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:54:40.498694326Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=info ts=2026-05-05T12:54:40.885962616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:54:40.889602378Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:54:42.164914816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:54:43.655699898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:54:43.663210912Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T12:54:43.83174348Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-05T12:54:45.430280797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:54:45.432830184Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:54:47.164678763Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:54:47.16469955Z caller=checkpoint.go:384 msg="flushing series" totalSize="750 kB" series=4
level=debug ts=2026-05-05T12:54:47.16471432Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:54:47.164725233Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:54:47.164739176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:54:47.172225286Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-05T12:54:47.173924797Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009260822s
level=info ts=2026-05-05T12:54:47.931610842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:54:47.939485419Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:54:48.14610281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:54:48.148940172Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-05T12:54:48.394759917Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52304"
ts=2026-05-05T12:54:49.283043211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:54:50.562260091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:54:50.579040787Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:54:50.579089108Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:54:50.579121282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:54:50.585578105Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T12:54:52.164351525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-05T12:54:57.164175641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:54:58.775909523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:54:58.779998816Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:54:58.780021941Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:54:58.780048785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:54:58.782711363Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T12:54:59.426834713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:49836"
level=info ts=2026-05-05T12:55:00.091983014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:55:00.097026522Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:55:00.17248221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:55:00.179601973Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:55:02.164786982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:55:02.497695656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:55:02.501313943Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T12:55:04.098746128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:55:04.101603738Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:55:05.869650943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:55:05.872839355Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:55:05.902941734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:55:05.910020807Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T12:55:06.311042756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:55:06.317524701Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:55:06.362584339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:55:06.602194579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:55:06.605826075Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:55:07.164343408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:55:09.097198121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:55:09.103781851Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:55:10.401840944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:55:10.404588461Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:55:11.572304373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:55:11.575611827Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=info ts=2026-05-05T12:55:11.745594013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:55:11.748742001Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:55:12.165120013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:55:12.253441922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:55:12.256022741Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:55:13.904853381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:55:13.9077831Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:55:15.778326769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:55:15.781735311Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:55:16.403652966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:55:16.410878131Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:55:16.410906326Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:55:16.410937135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:55:16.417617248Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:55:17.143933762Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:55:17.143962695Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:55:17.164325559Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-05T12:55:17.164346775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:55:17.164373737Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-05T12:55:17.164365803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:55:17.164384003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T12:55:17.164483484Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-05T12:55:17.164490098Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T12:55:17.164520117Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=24.276µs success=true
level=debug ts=2026-05-05T12:55:17.164751915Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-05T12:55:18.319733473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:55:18.322762038Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:55:18.332902275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T12:55:18.335270814Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-05T12:55:19.284782444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=info ts=2026-05-05T12:55:19.89044553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:55:19.893010485Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:55:20.499513722Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=warn ts=2026-05-05T12:55:22.164128199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:55:23.832186144Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-05T12:55:24.134037647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:55:24.13763343Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-05T12:55:26.370508251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:56454"
level=warn ts=2026-05-05T12:55:27.16510752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:55:27.165294999Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-05T12:55:27.165309855Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-05T12:55:29.42975247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:51944"
level=info ts=2026-05-05T12:55:29.631032174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:55:29.634013106Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:55:30.381377639Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:55:30.498433311Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-05T12:55:31.23096126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:55:31.240028136Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:55:32.142701054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:55:32.146253393Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T12:55:32.164322704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:55:33.033339393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:55:33.035889355Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:55:33.832227461Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-05T12:55:37.165040556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:55:37.165409185Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=debug ts=2026-05-05T12:55:38.820922066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:55:38.822654734Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-05T12:55:40.498619531Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-05-05T12:55:42.16433531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:55:43.289109211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:55:43.295580272Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T12:55:43.832882245Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=debug ts=2026-05-05T12:55:43.832908266Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T12:55:46.524749792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:55:46.528095739Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:55:47.164026182Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:55:47.164084361Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:55:47.16409381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:55:47.164100392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:55:47.165274926Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-05T12:55:48.297956714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:55:48.301607212Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:55:49.008965041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:55:49.011725219Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-05T12:55:49.287873504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:55:49.504930801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:55:49.509318174Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:55:49.985793946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:55:49.989895456Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:55:50.49870583Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=info ts=2026-05-05T12:55:50.918013414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:55:50.924423217Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:55:52.165084694Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:55:53.050952992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:55:53.054526991Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=debug ts=2026-05-05T12:55:53.831808608Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T12:55:54.760576145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:55:54.764305691Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:55:54.873481187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:55:54.880590121Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:55:55.208795272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:55:55.211275439Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:55:55.645569279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:55:55.651902803Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:55:57.164297507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:55:57.168480763Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:55:57.16850331Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=info ts=2026-05-05T12:55:57.964725315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:55:57.967255638Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:55:59.844014075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:55:59.85040832Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:56:00.498395425Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=debug ts=2026-05-05T12:56:01.902488171Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:56:02.164778317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:03.833142904Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=info ts=2026-05-05T12:56:04.265774546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:56:04.26879575Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:56:04.76930709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T12:56:04.77211068Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T12:56:04.969775352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:56:04.972868975Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-05T12:56:07.164997236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:07.165138152Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-05T12:56:10.025045395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:56:10.029218132Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:56:10.498700033Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-05T12:56:12.165031529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:56:13.371110428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:56:13.374681Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:56:13.832109804Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-05T12:56:13.832138562Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-05-05T12:56:17.143698107Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:56:17.144764963Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:56:17.164159668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:56:17.164191525Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:56:17.164202177Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:56:17.16423628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:17.165339855Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-05T12:56:18.287711689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:56:18.290616162Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:56:19.055171405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:56:19.058048232Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-05T12:56:19.290863679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:56:19.836865546Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:56:20.500892978Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-05T12:56:20.500935474Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=2
level=warn ts=2026-05-05T12:56:22.165132079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:23.832208451Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T12:56:25.448206165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:56:25.451848521Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:56:25.965324478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:56:25.967817055Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:56:25.96783749Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:56:25.967859539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:56:25.969953335Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T12:56:27.164834938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:27.236814266Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-05T12:56:27.23685937Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=2
level=info ts=2026-05-05T12:56:29.320027248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:56:29.323522361Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:56:29.726613335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:56:29.730081218Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-05T12:56:30.499260263Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-05T12:56:30.565269706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:56:30.568590953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:56:30.570306076Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:56:30.576879221Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T12:56:32.164480173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:33.831654701Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-05T12:56:35.473487824Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:56:35.473531937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9c6df0a56f5a68c3 immediate=false num_chunks=1 total_comp="174 kB" avg_comp="174 kB" total_uncomp="894 kB" avg_uncomp="894 kB" idle=1 labels="{stream=\"integration2-qbel\"}"
level=error ts=2026-05-05T12:56:35.477362449Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9c6df0a56f5a68c3/19df7faad9f%3A19df7feff18%3A3b2a367?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-qbel\"}"
level=info ts=2026-05-05T12:56:36.538548036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:56:36.542535021Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T12:56:37.164417462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:37.165538025Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-05T12:56:37.980484347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:56:37.983112819Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:56:39.065537389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:56:39.068201792Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:56:40.248100106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:56:40.252847555Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=debug ts=2026-05-05T12:56:40.498620859Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-05T12:56:40.720212175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:56:40.723864595Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:56:42.164669873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:43.831543813Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-05T12:56:45.674219772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T12:56:45.676833236Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T12:56:46.85473611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:56:46.857482536Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:56:47.16485201Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:56:47.164862789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:47.164882463Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:56:47.164891821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:56:47.164937139Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-05T12:56:47.50106624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:56:47.509541782Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-05T12:56:48.403597178Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39098"
ts=2026-05-05T12:56:49.29286915Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T12:56:50.499919169Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T12:56:50.499944306Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-05T12:56:51.836126643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:56:51.84309697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9c6df0a56f5a68c3 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"integration2-qbel\"}"
level=error ts=2026-05-05T12:56:51.845668289Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:56:51.845670903Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9c6df0a56f5a68c3/19df7faad9f%3A19df7feff18%3A3b2a367?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-qbel\"}"
level=warn ts=2026-05-05T12:56:52.164738892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:56:53.242113536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:56:53.249955539Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:56:53.832147509Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-05T12:56:56.862196526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:56:56.867887428Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-05T12:56:57.165109103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:56:57.165841837Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=info ts=2026-05-05T12:56:57.229323429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:56:57.232648284Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:56:57.552041421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:56:57.554629656Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:56:57.942260695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:56:57.945250454Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:56:58.995192496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:56:59.003498089Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:56:59.697584903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:56:59.700767053Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-05T12:57:00.498572182Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-05-05T12:57:02.164582437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:57:02.91819051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:57:02.920983911Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:57:03.396334021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:57:03.451514998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:57:03.45420678Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:57:03.832353568Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-05T12:57:04.95088523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T12:57:07.164362345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:07.166026887Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-05T12:57:07.166044066Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-05T12:57:08.367990454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:57:08.370970235Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:57:09.434175653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:57:09.436851404Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:57:10.516237945Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-05T12:57:10.516263397Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=1
level=info ts=2026-05-05T12:57:11.07152423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:57:11.075771321Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:57:11.07581175Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:57:11.075844867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:57:11.079527195Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:57:12.164963097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:13.832248702Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=info ts=2026-05-05T12:57:15.014700166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:57:15.021345752Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:57:16.868164513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9c6df0a56f5a68c3 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"integration2-qbel\"}"
level=error ts=2026-05-05T12:57:16.871163023Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9c6df0a56f5a68c3/19df7faad9f%3A19df7feff18%3A3b2a367?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-qbel\"}"
level=info ts=2026-05-05T12:57:17.143996685Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T12:57:17.1440068Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:57:17.164459881Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:57:17.164447358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:17.164488689Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:57:17.164499878Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:57:17.165588598Z caller=checkpoint.go:372 msg="writing series" size="9.7 kB"
ts=2026-05-05T12:57:19.295713007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T12:57:20.494505204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:57:20.497667892Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:57:20.498710705Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=warn ts=2026-05-05T12:57:22.165145465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:23.83238506Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-05T12:57:24.613644798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:57:24.617126893Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-05T12:57:26.380870361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:50184"
level=info ts=2026-05-05T12:57:27.053114155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:57:27.057924617Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:57:27.057953775Z caller=flush.go:240 component=ingester loop=8 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:57:27.057980132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:57:27.061017233Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=warn ts=2026-05-05T12:57:27.16444148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:27.165664713Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-05T12:57:28.028316788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T12:57:28.031746409Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T12:57:28.064810128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:57:28.068485617Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:57:28.068508856Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:57:28.068536519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:57:28.071382837Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:57:28.942311336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:57:28.945707147Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:57:29.415105838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:57:29.418594447Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-05T12:57:29.440981052Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36738"
level=info ts=2026-05-05T12:57:29.584767837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:57:29.588520967Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T12:57:30.414548812Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:57:30.499148473Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-05T12:57:32.164661349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:33.83152935Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-05T12:57:35.852569666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:57:35.860553522Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-05T12:57:37.164599785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:37.165952286Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-05T12:57:40.499351169Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-05T12:57:40.518134838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:57:40.526749659Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:57:42.164331975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:57:42.661212253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:57:42.666874101Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-05T12:57:43.832047228Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-05T12:57:43.832091369Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=10
level=info ts=2026-05-05T12:57:44.146792633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:57:44.155661227Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:57:44.165660988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:57:44.168568928Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:57:44.614695408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:57:44.61740115Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:57:46.561408065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:57:46.566806018Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=info ts=2026-05-05T12:57:46.568820465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:57:46.575556768Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:57:46.98477518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:57:46.987455906Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T12:57:47.164760843Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:57:47.164796942Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:57:47.164808358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:57:47.164819505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:47.165493138Z caller=checkpoint.go:372 msg="writing series" size="651 kB"
level=info ts=2026-05-05T12:57:47.442500588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:57:47.445308694Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-05T12:57:48.408790435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60944"
ts=2026-05-05T12:57:49.298818752Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T12:57:50.50001129Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-05T12:57:50.500035373Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=2
level=warn ts=2026-05-05T12:57:52.164736242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:53.832517956Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-05T12:57:54.189550077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:57:54.193110649Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:57:54.521933056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:57:54.524603382Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:57:54.589907806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:57:54.592578405Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:57:55.455188035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:57:55.461256235Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T12:57:57.164202242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:57:57.165327349Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-05T12:57:58.936723544Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-05T12:58:00.498793381Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=warn ts=2026-05-05T12:58:02.164923368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:58:02.869194611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:58:02.872646738Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:58:02.884695023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:58:02.887192002Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:58:03.130861703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:58:03.133581594Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:58:03.831576958Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-05T12:58:05.287108399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9c6df0a56f5a68c3 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"integration2-qbel\"}"
level=error ts=2026-05-05T12:58:05.290167395Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9c6df0a56f5a68c3/19df7faad9f%3A19df7feff18%3A3b2a367?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-qbel\"}"
level=debug ts=2026-05-05T12:58:05.433863109Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:58:05.693437547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:58:05.697186833Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-05T12:58:07.165115003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:58:07.165295537Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=info ts=2026-05-05T12:58:08.150399794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:58:08.153743788Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:58:08.697110771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:58:08.69977096Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:58:08.905256552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:58:08.907944588Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:58:10.498956022Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-05T12:58:12.164797523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:58:13.831777655Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-05T12:58:17.14023742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=debug ts=2026-05-05T12:58:17.14365128Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:58:17.14389245Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=error ts=2026-05-05T12:58:17.14436617Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=info ts=2026-05-05T12:58:17.164654825Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:58:17.164669252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:58:17.164681729Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:58:17.164712014Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T12:58:17.164834961Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
ts=2026-05-05T12:58:19.3013375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T12:58:20.499599648Z caller=checkpoint.go:372 msg="writing series" size="545 kB"
level=debug ts=2026-05-05T12:58:20.499623159Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-05-05T12:58:21.60274275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:58:21.606327687Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T12:58:22.1648101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:58:23.832296778Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-05T12:58:25.37207009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:58:25.377577205Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T12:58:27.164978377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:58:27.167101629Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T12:58:27.167117311Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-05-05T12:58:27.417463643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T12:58:27.421612316Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-05T12:58:30.498863696Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=info ts=2026-05-05T12:58:31.841195365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:58:31.844549155Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T12:58:32.002028685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:58:32.009347376Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T12:58:32.164690357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:58:32.811710094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:58:32.819233465Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T12:58:33.711927937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:58:33.718450683Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T12:58:33.83184323Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-05T12:58:34.141786184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:58:34.14913949Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:58:37.164177615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:58:37.165440025Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-05T12:58:37.829774752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:58:37.832636482Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T12:58:38.728112239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72c194fe60774727 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="198 kB" avg_uncomp="198 kB" idle=1 labels="{stream=\"build-e2e-uctj\"}"
level=debug ts=2026-05-05T12:58:38.728151995Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-05T12:58:38.73118416Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72c194fe60774727/19df7ede838%3A19df800f703%3A676b2122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uctj\"}"
level=info ts=2026-05-05T12:58:38.755409415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:58:38.761790468Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:58:39.098483233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="548 B" avg_comp="548 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:58:39.101077266Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T12:58:40.499011768Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:58:40.982276768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:58:40.984960138Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:58:40.984980454Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:58:40.985000401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:58:40.987042953Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-05T12:58:42.164877661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:58:42.760225219Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T12:58:42.973785084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:58:42.976638683Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:58:42.976660386Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:58:42.97668506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:58:42.978914961Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:58:43.331113775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:58:43.333767749Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:58:43.781044384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:58:43.784602021Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-05T12:58:43.784627191Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:58:43.784652486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:58:43.787451366Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T12:58:43.831830628Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-05-05T12:58:47.164076312Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:58:47.164105471Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:58:47.16411414Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:58:47.164101439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:58:47.165276759Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
ts=2026-05-05T12:58:49.303290745Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:58:49.901501483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9c6df0a56f5a68c3 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"integration2-qbel\"}"
level=error ts=2026-05-05T12:58:49.905633931Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9c6df0a56f5a68c3/19df7faad9f%3A19df7feff18%3A3b2a367?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-qbel\"}"
level=info ts=2026-05-05T12:58:50.205014114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:58:50.208190335Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T12:58:50.49923461Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-05T12:58:50.499258398Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-05T12:58:52.164801588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:58:52.651872414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72c194fe60774727 immediate=false num_chunks=1 total_comp="56 kB" avg_comp="56 kB" total_uncomp="198 kB" avg_uncomp="198 kB" full=1 labels="{stream=\"build-e2e-uctj\"}"
level=error ts=2026-05-05T12:58:52.65495003Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72c194fe60774727/19df7ede838%3A19df800f703%3A676b2122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uctj\"}"
level=info ts=2026-05-05T12:58:53.453977647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:58:53.456364094Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T12:58:53.58477668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:58:53.587330497Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=debug ts=2026-05-05T12:58:53.831956273Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-05-05T12:58:54.056629205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:58:54.059882518Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T12:58:54.790574785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:58:54.793338696Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T12:58:56.196338282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:58:56.199209714Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:58:56.199236236Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:58:56.199266046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:58:56.201278573Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T12:58:57.164174214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:58:57.172988451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:58:57.179124321Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:58:57.179155602Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:58:57.179185558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:58:57.181214265Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-05T12:58:57.181230446Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=error ts=2026-05-05T12:58:57.181961034Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:58:59.394007686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:58:59.399390394Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:58:59.404419604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:58:59.408116696Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:58:59.808195106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:58:59.811230858Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:59:00.498683037Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-05T12:59:02.164840224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:59:03.832147157Z caller=checkpoint.go:372 msg="writing series" size="786 kB"
level=info ts=2026-05-05T12:59:04.943887002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T12:59:04.947595795Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=warn ts=2026-05-05T12:59:07.164484676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:59:07.168604368Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-05-05T12:59:07.168625467Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.7 MB" series=3
level=debug ts=2026-05-05T12:59:08.961374634Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T12:59:10.498378456Z caller=checkpoint.go:372 msg="writing series" size="895 B"
level=info ts=2026-05-05T12:59:11.145639088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T12:59:11.148920116Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-05T12:59:12.164756061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:59:13.63887571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:59:13.641467451Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-05T12:59:13.832070082Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=info ts=2026-05-05T12:59:15.416254349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:59:15.41972047Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T12:59:15.830709521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:59:15.833623036Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T12:59:17.143854857Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T12:59:17.143895199Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T12:59:17.164236144Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T12:59:17.164263944Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:59:17.164272593Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T12:59:17.164276058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:59:17.167806176Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-05T12:59:17.167830765Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=3
ts=2026-05-05T12:59:18.415442927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39080"
level=info ts=2026-05-05T12:59:19.049349184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T12:59:19.052789904Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-05T12:59:19.305120384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=info ts=2026-05-05T12:59:19.338041543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T12:59:19.34545461Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T12:59:20.499318929Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=info ts=2026-05-05T12:59:20.514251609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T12:59:20.516806336Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T12:59:21.126156634Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-05T12:59:21.12782631Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-05T12:59:21.216533448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72c194fe60774727 immediate=false num_chunks=1 total_comp="56 kB" avg_comp="56 kB" total_uncomp="198 kB" avg_uncomp="198 kB" full=1 labels="{stream=\"build-e2e-uctj\"}"
level=error ts=2026-05-05T12:59:21.219050248Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72c194fe60774727/19df7ede838%3A19df800f703%3A676b2122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uctj\"}"
level=info ts=2026-05-05T12:59:21.952633094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T12:59:21.960007019Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T12:59:22.164454351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:59:22.87888574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T12:59:22.886856625Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T12:59:23.74617046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T12:59:23.748829862Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-05T12:59:23.832415957Z caller=checkpoint.go:372 msg="writing series" size="695 kB"
level=info ts=2026-05-05T12:59:24.108579375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T12:59:24.11583125Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T12:59:25.28151304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:59:25.28841148Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:59:25.288432758Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:59:25.288460914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T12:59:25.294111756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:59:25.294160993Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:59:25.295869509Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:59:25.295879159Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:59:25.588751887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:59:25.591157875Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-05T12:59:26.390314311Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34944"
level=info ts=2026-05-05T12:59:26.692815729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T12:59:26.69531672Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-05T12:59:27.164126985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:59:27.165185015Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-05-05T12:59:30.500037449Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-05T12:59:30.500079518Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=4
level=warn ts=2026-05-05T12:59:32.165095225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:59:32.402914053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:59:32.405628723Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-05T12:59:32.405648908Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:59:32.405672251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:59:32.407816564Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-05T12:59:33.831576891Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=debug ts=2026-05-05T12:59:33.999245427Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T12:59:35.328767455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T12:59:35.331947869Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T12:59:35.977921499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:59:35.980518984Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:59:35.980544729Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:59:35.980568378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:59:35.982511089Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-05T12:59:36.900858709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T12:59:36.904316865Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-05T12:59:37.164948574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:59:37.164995261Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=info ts=2026-05-05T12:59:38.032538955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:59:38.035698625Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:59:38.035717587Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:59:38.035740706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:59:38.038386482Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:59:40.207133516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T12:59:40.215303727Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-05T12:59:40.499421046Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=info ts=2026-05-05T12:59:41.786259042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:59:41.788921788Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:59:41.788939449Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:59:41.788959279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:59:41.792673017Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=warn ts=2026-05-05T12:59:42.164730122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:59:42.377275809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T12:59:42.379706168Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T12:59:42.493944884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T12:59:42.496394374Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T12:59:42.724181673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T12:59:42.726624848Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T12:59:42.813988982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9c6df0a56f5a68c3 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"integration2-qbel\"}"
level=error ts=2026-05-05T12:59:42.81716392Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9c6df0a56f5a68c3/19df7faad9f%3A19df7feff18%3A3b2a367?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-qbel\"}"
level=debug ts=2026-05-05T12:59:43.832098457Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-05T12:59:46.041535205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:59:46.044321901Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:59:46.044343006Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T12:59:46.044366765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:59:46.046551088Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-05T12:59:47.164905983Z caller=checkpoint.go:384 msg="flushing series" totalSize="800 kB" series=4
level=info ts=2026-05-05T12:59:47.164943265Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T12:59:47.164943614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T12:59:47.164966478Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T12:59:47.164975422Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T12:59:47.170903249Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-05T12:59:47.173835984Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009152938s
ts=2026-05-05T12:59:48.417195979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:37522"
level=info ts=2026-05-05T12:59:49.267261951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T12:59:49.269877664Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-05T12:59:49.306990408Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=warn ts=2026-05-05T12:59:52.164705982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:59:52.229885152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T12:59:52.234265502Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T12:59:54.479848339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T12:59:54.482634689Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T12:59:55.644779512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-05T12:59:56.392273092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:37068"
level=info ts=2026-05-05T12:59:57.120354024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T12:59:57.122847119Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T12:59:57.164758633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T12:59:57.88094004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T12:59:57.884769846Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T12:59:58.130351972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T12:59:58.133647295Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T12:59:59.78348653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T12:59:59.786820374Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T13:00:02.164127714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:00:02.654118778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T13:00:02.660628181Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=info ts=2026-05-05T13:00:03.018189472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T13:00:03.021087553Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T13:00:03.165717118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T13:00:03.169235329Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T13:00:05.733105998Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T13:00:05.987921155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T13:00:05.990917984Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-05T13:00:05.990941556Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T13:00:05.990964027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T13:00:05.993462239Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T13:00:06.018528088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T13:00:06.021102901Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-05T13:00:07.165018467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:00:07.597504317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T13:00:07.60163435Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T13:00:11.998404217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T13:00:12.001014829Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-05T13:00:12.164427183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:00:12.901628525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T13:00:12.904646462Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T13:00:13.311261303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T13:00:13.317332137Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T13:00:14.82269932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T13:00:14.825860386Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T13:00:15.203068355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T13:00:15.210834891Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T13:00:15.793035756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72c194fe60774727 immediate=false num_chunks=1 total_comp="56 kB" avg_comp="56 kB" total_uncomp="198 kB" avg_uncomp="198 kB" full=1 labels="{stream=\"build-e2e-uctj\"}"
level=error ts=2026-05-05T13:00:15.796140154Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72c194fe60774727/19df7ede838%3A19df800f703%3A676b2122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uctj\"}"
level=info ts=2026-05-05T13:00:16.701659064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T13:00:16.709607292Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T13:00:17.143887147Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T13:00:17.143917978Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=debug ts=2026-05-05T13:00:17.145239789Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-05T13:00:17.14528579Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1975539 n=1
level=debug ts=2026-05-05T13:00:17.145326833Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1975539 n=1
level=info ts=2026-05-05T13:00:17.164253184Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-05T13:00:17.164261328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:00:17.164274828Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T13:00:17.164294177Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T13:00:17.164302861Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-05T13:00:17.164393946Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-05T13:00:17.164402088Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-05T13:00:17.164418637Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.153µs success=true
level=debug ts=2026-05-05T13:00:17.164662596Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-05T13:00:17.591569404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T13:00:17.599042297Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T13:00:18.280934892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T13:00:18.283742681Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-05T13:00:18.418919977Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:58412"
ts=2026-05-05T13:00:19.309860569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-49342479 10.130.0.63:7946"
level=debug ts=2026-05-05T13:00:20.499405563Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=info ts=2026-05-05T13:00:20.61095645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T13:00:20.618713674Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T13:00:22.164279418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:00:23.831411648Z caller=checkpoint.go:372 msg="writing series" size="1.1 kB"
level=info ts=2026-05-05T13:00:24.683965913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9c6df0a56f5a68c3 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"integration2-qbel\"}"
level=error ts=2026-05-05T13:00:24.688658847Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9c6df0a56f5a68c3/19df7faad9f%3A19df7feff18%3A3b2a367?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-qbel\"}"
level=warn ts=2026-05-05T13:00:27.164324034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:00:27.165393883Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=info ts=2026-05-05T13:00:27.665474998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T13:00:27.669513566Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-05T13:00:29.458959876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:36446"
level=debug ts=2026-05-05T13:00:30.498876498Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-05T13:00:30.498900614Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T13:00:30.649993202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T13:00:30.654296103Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T13:00:31.723626174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T13:00:31.726658794Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-05T13:00:32.164548011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:00:32.257816243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T13:00:32.261366837Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-05T13:00:33.832247025Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-05T13:00:34.699554436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T13:00:34.702614382Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-05T13:00:37.164120052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:00:37.165614676Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=debug ts=2026-05-05T13:00:40.49901185Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=info ts=2026-05-05T13:00:40.505954562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T13:00:40.509495105Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-05T13:00:40.54894749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T13:00:40.551766835Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-05T13:00:40.81171655Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T13:00:42.165159761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:00:43.832736218Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=debug ts=2026-05-05T13:00:43.832757612Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-05T13:00:45.134751634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T13:00:45.137934223Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T13:00:46.598513696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T13:00:46.602622244Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T13:00:47.164063347Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T13:00:47.16411353Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T13:00:47.164124528Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T13:00:47.164120348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:00:47.165318757Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
ts=2026-05-05T13:00:48.420170341Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52324"
level=info ts=2026-05-05T13:00:49.187345553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="515 B" avg_comp="515 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T13:00:49.189954118Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-05T13:00:49.312307028Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T13:00:50.49820848Z caller=checkpoint.go:372 msg="writing series" size="653 B"
level=debug ts=2026-05-05T13:00:51.355262998Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T13:00:51.962741105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T13:00:51.965560102Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-05T13:00:52.164561994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:00:53.080181092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T13:00:53.083746096Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-05T13:00:53.083770655Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T13:00:53.083799112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T13:00:53.086773824Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-05T13:00:53.832108205Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-05T13:00:53.949472598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T13:00:53.951938295Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T13:00:54.57496369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T13:00:54.577536062Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T13:00:55.03986668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T13:00:55.043224989Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T13:00:56.820801097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T13:00:56.823541964Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-05T13:00:57.16432615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:00:57.167036168Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-05T13:00:57.167073439Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=4
level=info ts=2026-05-05T13:00:59.258268839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T13:00:59.264947172Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=error ts=2026-05-05T13:00:59.26498187Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T13:00:59.265025065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T13:00:59.271225493Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T13:00:59.426747045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T13:00:59.43361631Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T13:00:59.433648899Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T13:00:59.566116896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T13:00:59.572022148Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T13:00:59.736345675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T13:00:59.739985049Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=info ts=2026-05-05T13:01:00.163823156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T13:01:00.166654259Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T13:01:00.49904673Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-05T13:01:01.931806699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T13:01:01.938468909Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T13:01:01.938491163Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T13:01:01.93853705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T13:01:01.94462951Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-05T13:01:02.164540378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:01:02.810350881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72c194fe60774727 immediate=false num_chunks=1 total_comp="56 kB" avg_comp="56 kB" total_uncomp="198 kB" avg_uncomp="198 kB" full=1 labels="{stream=\"build-e2e-uctj\"}"
level=error ts=2026-05-05T13:01:02.813335414Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72c194fe60774727/19df7ede838%3A19df800f703%3A676b2122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uctj\"}"
level=debug ts=2026-05-05T13:01:03.831666061Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=info ts=2026-05-05T13:01:04.650713863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T13:01:04.654764362Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-05T13:01:05.016755277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9c6df0a56f5a68c3 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"integration2-qbel\"}"
level=error ts=2026-05-05T13:01:05.019595946Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9c6df0a56f5a68c3/19df7faad9f%3A19df7feff18%3A3b2a367?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-qbel\"}"
level=warn ts=2026-05-05T13:01:07.164700986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:01:07.16483256Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=debug ts=2026-05-05T13:01:09.353454497Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T13:01:10.121575071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T13:01:10.128170532Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=debug ts=2026-05-05T13:01:10.498306119Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-05T13:01:10.573745279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T13:01:10.581203293Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-05T13:01:10.857123718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T13:01:10.86388533Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-05T13:01:12.164670635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:01:13.452518602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T13:01:13.459160927Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-05T13:01:13.832378429Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-05T13:01:13.832411808Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-05T13:01:14.989705025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T13:01:14.992876737Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-05T13:01:15.963381095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T13:01:15.971235868Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-05T13:01:17.144109493Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-05T13:01:17.14411529Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T13:01:17.164540424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-05T13:01:17.164542849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:01:17.164562427Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T13:01:17.164571739Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-05T13:01:17.164736847Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
ts=2026-05-05T13:01:18.421986816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:40362"
ts=2026-05-05T13:01:19.315174481Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=info ts=2026-05-05T13:01:19.328611875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T13:01:19.331294764Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T13:01:20.500915652Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-05T13:01:20.50093755Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=2
level=warn ts=2026-05-05T13:01:22.16462679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:01:23.831555938Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-05T13:01:24.248969943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T13:01:24.252996527Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-05T13:01:26.398764031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.63:34678"
level=info ts=2026-05-05T13:01:26.762111434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T13:01:26.766932825Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-05T13:01:26.858175769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T13:01:26.860975254Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-05T13:01:27.164176923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:01:27.24401807Z caller=checkpoint.go:372 msg="writing series" size="63 MB"
level=debug ts=2026-05-05T13:01:27.244048482Z caller=checkpoint.go:384 msg="flushing series" totalSize="63 MB" series=2
level=info ts=2026-05-05T13:01:27.470025495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T13:01:27.473612986Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-05T13:01:30.498471391Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=info ts=2026-05-05T13:01:31.019767292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T13:01:31.030236812Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T13:01:31.410933423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T13:01:31.418825Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-05T13:01:32.050069795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T13:01:32.05450929Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T13:01:32.164418657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:01:33.831405822Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-05-05T13:01:33.928639625Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-05T13:01:36.107329151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T13:01:36.11055847Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T13:01:36.13166514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T13:01:36.134148354Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-05T13:01:36.95779811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T13:01:36.961564196Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T13:01:37.164188225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:01:37.165271946Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-05T13:01:37.39411771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T13:01:37.39761332Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-05T13:01:40.135041781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T13:01:40.137702636Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T13:01:40.170899608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T13:01:40.174354873Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T13:01:40.287605584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T13:01:40.290310011Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-05T13:01:40.49907463Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-05T13:01:41.260272924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T13:01:41.263204876Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=warn ts=2026-05-05T13:01:42.164200626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:01:43.089913806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72c194fe60774727 immediate=false num_chunks=1 total_comp="56 kB" avg_comp="56 kB" total_uncomp="198 kB" avg_uncomp="198 kB" full=1 labels="{stream=\"build-e2e-uctj\"}"
level=error ts=2026-05-05T13:01:43.092695248Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72c194fe60774727/19df7ede838%3A19df800f703%3A676b2122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uctj\"}"
level=info ts=2026-05-05T13:01:43.82377605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T13:01:43.827470361Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=debug ts=2026-05-05T13:01:43.831556621Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-05T13:01:44.831314013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T13:01:44.894393644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T13:01:44.897730159Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T13:01:44.897754313Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T13:01:44.897784808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T13:01:44.900542102Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-05T13:01:47.16404616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T13:01:47.164100778Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T13:01:47.164110255Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T13:01:47.164098262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:01:47.165314296Z caller=checkpoint.go:372 msg="writing series" size="6.0 kB"
ts=2026-05-05T13:01:49.317378623Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-dckzh-97a0b90a 10.128.0.89:7946"
level=debug ts=2026-05-05T13:01:50.49937469Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-05T13:01:50.49939837Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-05T13:01:50.794573896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T13:01:50.797309968Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-05T13:01:50.855536948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="516 MB" avg_uncomp="23 MB" full=20 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-05T13:01:50.862627463Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19df7e2985c%3A19df7ebde8e%3Ad37c65f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-05T13:01:51.186110243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="138 MB" avg_uncomp="23 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-05T13:01:51.193259699Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19df7e2a141%3A19df7e3389e%3Afc15b34b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-05T13:01:52.165114846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:01:52.730501458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9c6df0a56f5a68c3 immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="894 kB" avg_uncomp="894 kB" full=1 labels="{stream=\"integration2-qbel\"}"
level=error ts=2026-05-05T13:01:52.733615947Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9c6df0a56f5a68c3/19df7faad9f%3A19df7feff18%3A3b2a367?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-qbel\"}"
level=debug ts=2026-05-05T13:01:53.831890951Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-05-05T13:01:57.164762383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:01:57.165661427Z caller=checkpoint.go:372 msg="writing series" size="569 kB"
level=info ts=2026-05-05T13:01:58.654182337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T13:01:58.658290995Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-05T13:01:59.467099531Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:57350"
level=info ts=2026-05-05T13:02:00.308664408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T13:02:00.31631795Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T13:02:00.316355387Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T13:02:00.31638998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T13:02:00.324125357Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-05T13:02:00.498496893Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-05T13:02:00.956217302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="455 kB" avg_comp="455 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-05T13:02:00.959706518Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19df7e2999a%3A19df811db3b%3A808e74a3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-05T13:02:02.1651019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:02:03.831420944Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-05T13:02:07.16482839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:02:07.165404046Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=debug ts=2026-05-05T13:02:07.1654203Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-05T13:02:07.252130985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="430 kB" avg_comp="430 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-05T13:02:07.256446165Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19df7e29930%3A19df8208d11%3Ac90d5212?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-05T13:02:08.088212026Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-05T13:02:10.516032021Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-05T13:02:10.516074366Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=1
level=warn ts=2026-05-05T13:02:12.164487948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:02:12.176577136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c5979bfd175d4a1d immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="48 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-dlty\"}"
level=error ts=2026-05-05T13:02:12.185625715Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c5979bfd175d4a1d/19df7ebf030%3A19df7f7f18c%3Af29352f4?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-dlty\"}"
level=info ts=2026-05-05T13:02:13.18711993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.4 MB" avg_comp="2.4 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-05T13:02:13.19355651Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19df7e299b3%3A19df81b9b52%3A5877eade?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-05T13:02:13.831541566Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-05T13:02:14.18770336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="552 kB" avg_comp="552 kB" total_uncomp="5.8 MB" avg_uncomp="5.8 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-05T13:02:14.191601126Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19df7e2b09c%3A19df80d7e29%3A18234eef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-05T13:02:14.916478449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-05T13:02:14.919546389Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19df7e33cf1%3A19df7e353e4%3Af79d1f20?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-05T13:02:17.143690576Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.91:11211 count=1
level=info ts=2026-05-05T13:02:17.144736408Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-05T13:02:17.164040738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-05T13:02:17.164092132Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-05T13:02:17.164104204Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-05T13:02:17.164100168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:02:17.165251753Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-05T13:02:18.426259564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:60552"
level=info ts=2026-05-05T13:02:18.645409307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.5 MB" avg_comp="1.8 MB" total_uncomp="23 MB" avg_uncomp="11 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-05T13:02:18.653858911Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19df7e29a72%3A19df7fe3f58%3A1759de0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-05T13:02:19.319926534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-p6l2c-8a874c89 10.129.0.76:7946"
level=debug ts=2026-05-05T13:02:20.498731441Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-05T13:02:21.870656098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="637 kB" avg_comp="637 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-05T13:02:21.874383337Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19df7e29823%3A19df804b022%3A584f7ce6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-05T13:02:22.165136087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-05T13:02:23.179519847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-05T13:02:23.182009095Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19df7e2c0a2%3A19df7e3429e%3Ab426753a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-05T13:02:23.542992866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="4.1 kB" avg_uncomp="4.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-05T13:02:23.545636444Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19df7e2c0a2%3A19df7f6a663%3Aa6e6c8f2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-05T13:02:23.831480705Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-05T13:02:25.473151272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.2 kB" avg_comp="6.2 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-05T13:02:25.476298714Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19df7e2a09b%3A19df7e3429e%3A7a44d039?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-05T13:02:25.634982092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="379 kB" avg_comp="379 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-05T13:02:25.638208751Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19df7e29820%3A19df804cf81%3Ab96cd7bc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-05T13:02:26.748549135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="194 kB" avg_uncomp="194 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-05T13:02:26.75117477Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19df7e340ef%3A19df7e3451b%3A8f9144c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-05T13:02:27.164881274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:02:27.164984544Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-05T13:02:28.12417537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T13:02:28.12694294Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T13:02:28.126966803Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-05T13:02:28.126990512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-05T13:02:28.130909109Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19df7eb4c40%3A19df7ec9461%3Ad1eb2915?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-05T13:02:28.361803028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-05T13:02:28.364624091Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19df7e3429c%3A19df7e3451b%3A74fad494?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-05T13:02:29.469961737Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.76:37852"
level=debug ts=2026-05-05T13:02:30.498538738Z caller=checkpoint.go:372 msg="writing series" size="264 kB"
level=info ts=2026-05-05T13:02:30.732981734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="533 B" avg_comp="533 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-05T13:02:30.736034764Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19df7e34519%3A19df7e3451b%3A7d1969aa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-05T13:02:30.890514346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.3 MB" avg_comp="1.1 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-05T13:02:30.894150581Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19df7e2a023%3A19df7f15cca%3A1fc55959?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-05T13:02:32.164526059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:02:33.832222791Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-05T13:02:34.084970431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-05T13:02:34.188447377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="56 kB" avg_uncomp="56 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-05T13:02:34.191311594Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19df7e2a09e%3A19df7e3429e%3A89c83e74?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-05T13:02:34.883248275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a0caf2a80da8dc8 immediate=false num_chunks=1 total_comp="452 kB" avg_comp="452 kB" total_uncomp="2.1 MB" avg_uncomp="2.1 MB" full=1 labels="{stream=\"integration1-zlmp\"}"
level=error ts=2026-05-05T13:02:34.886942054Z caller=flush.go:262 component=ingester loop=8 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a0caf2a80da8dc8/19df7ec32b8%3A19df7fa1931%3Aee491599?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zlmp\"}"
level=debug ts=2026-05-05T13:02:36.655796212Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-05T13:02:37.164218478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-05T13:02:37.165439917Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-05-05T13:02:38.352749356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ad07f4012f0a40d7 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-zspa\"}"
level=error ts=2026-05-05T13:02:38.355403211Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ad07f4012f0a40d7/19df7ec943f%3A19df7ee8091%3Ac7b30911?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zspa\"}"
level=info ts=2026-05-05T13:02:39.463676275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="566 B" avg_comp="566 B" total_uncomp="598 B" avg_uncomp="598 B" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-05T13:02:39.467547662Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19df7e2c0a2%3A19df7e2c0a3%3Abba83281?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-05T13:02:40.498608898Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-05-05T13:02:40.498630964Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-05-05T13:02:40.989096124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72c194fe60774727 immediate=false num_chunks=1 total_comp="56 kB" avg_comp="56 kB" total_uncomp="198 kB" avg_uncomp="198 kB" full=1 labels="{stream=\"build-e2e-uctj\"}"
level=error ts=2026-05-05T13:02:40.992018841Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72c194fe60774727/19df7ede838%3A19df800f703%3A676b2122?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-uctj\"}"
level=warn ts=2026-05-05T13:02:42.165004275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"